{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Read Data Sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:06.838312Z",
     "start_time": "2017-07-24T00:03:06.422071Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from collections import namedtuple\n",
    "pd.set_option(\"display.max_rows\",100)\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:06.849901Z",
     "start_time": "2017-07-24T00:03:06.839855Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "rm dataset/scores/tf_dense_only_nsl_kdd_scores_all-.pkl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:06.917958Z",
     "start_time": "2017-07-24T00:03:06.851394Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class dataset:\n",
    "    kdd_train_2labels = pd.read_pickle(\"dataset/kdd_train__2labels.pkl\")\n",
    "    kdd_test_2labels = pd.read_pickle(\"dataset/kdd_test_2labels.pkl\")\n",
    "    kdd_test__2labels = pd.read_pickle(\"dataset/kdd_test__2labels.pkl\")\n",
    "\n",
    "    kdd_train_5labels = pd.read_pickle(\"dataset/kdd_train_5labels.pkl\")\n",
    "    kdd_test_5labels = pd.read_pickle(\"dataset/kdd_test_5labels.pkl\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:06.924906Z",
     "start_time": "2017-07-24T00:03:06.919523Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25192, 124)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_train_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:06.933823Z",
     "start_time": "2017-07-24T00:03:06.926260Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(22544, 124)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.kdd_test_2labels.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:07.331348Z",
     "start_time": "2017-07-24T00:03:06.935191Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(25192, 122)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import model_selection as ms\n",
    "from sklearn import preprocessing as pp\n",
    "\n",
    "class preprocess:\n",
    "    \n",
    "    output_columns_2labels = ['is_Normal','is_Attack']\n",
    "    \n",
    "    x_input = dataset.kdd_train_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_output = dataset.kdd_train_2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    x_test_input = dataset.kdd_test_2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test = dataset.kdd_test_2labels.loc[:,output_columns_2labels]\n",
    "    \n",
    "    x_test__input = dataset.kdd_test__2labels.drop(output_columns_2labels, axis = 1)\n",
    "    y_test_ = dataset.kdd_test__2labels.loc[:,output_columns_2labels]\n",
    "\n",
    "    ss = pp.StandardScaler()\n",
    "\n",
    "    x_train = ss.fit_transform(x_input)\n",
    "    x_test = ss.transform(x_test_input)\n",
    "    x_test_ = ss.transform(x_test__input)\n",
    "\n",
    "    y_train = y_output.values\n",
    "    y_test = y_test.values\n",
    "    y_test_ = y_test_.values\n",
    "\n",
    "preprocess.x_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:08.440070Z",
     "start_time": "2017-07-24T00:03:07.332880Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:08.629593Z",
     "start_time": "2017-07-24T00:03:08.441558Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class network(object):\n",
    "    \n",
    "    input_dim = 122\n",
    "    classes = 2\n",
    "    hidden_encoder_dim = 122\n",
    "    hidden_layers = 1\n",
    "    latent_dim = 18\n",
    "\n",
    "    def __init__(self, classes, hidden_layers, num_of_features):\n",
    "        self.classes = classes\n",
    "        self.hidden_layers = hidden_layers\n",
    "        self.latent_dim = num_of_features\n",
    "            \n",
    "    def build_layers(self):\n",
    "        tf.reset_default_graph()\n",
    "        #learning_rate = tf.Variable(initial_value=0.001)\n",
    "\n",
    "        input_dim = self.input_dim\n",
    "        classes = self.classes\n",
    "        hidden_encoder_dim = self.hidden_encoder_dim\n",
    "        hidden_layers = self.hidden_layers\n",
    "        latent_dim = self.latent_dim\n",
    "        \n",
    "        with tf.variable_scope(\"Input\"):\n",
    "            self.x = tf.placeholder(\"float\", shape=[None, input_dim])\n",
    "            self.y_ = tf.placeholder(\"float\", shape=[None, classes])\n",
    "            self.keep_prob = tf.placeholder(\"float\")\n",
    "            self.lr = tf.placeholder(\"float\")\n",
    "        \n",
    "        with tf.variable_scope(\"Layer_Encoder\"):\n",
    "\n",
    "            hidden_encoder = tf.layers.dense(self.x, hidden_encoder_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "            hidden_encoder = tf.nn.dropout(hidden_encoder, self.keep_prob)\n",
    "            for h in range(hidden_layers - 1):\n",
    "                hidden_encoder = tf.layers.dense(hidden_encoder, latent_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "                hidden_encoder = tf.nn.dropout(hidden_encoder, self.keep_prob)\n",
    "            \n",
    "            #hidden_encoder = tf.layers.dense(self.x, latent_dim, activation = tf.nn.relu, kernel_regularizer=tf.nn.l2_loss)\n",
    "            #hidden_encoder = tf.nn.dropout(hidden_encoder, self.keep_prob)\n",
    "            \n",
    "        with tf.variable_scope(\"Layer_Dense_Softmax\"):\n",
    "            self.y = tf.layers.dense(hidden_encoder, classes, activation=tf.nn.softmax)\n",
    "            \n",
    "        with tf.variable_scope(\"Loss\"):\n",
    "            \n",
    "            loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels = self.y_, logits = self.y))\n",
    "\n",
    "            #loss = tf.clip_by_value(loss, -1e-1, 1e-1)\n",
    "            #loss = tf.where(tf.is_nan(loss), 1e-1, loss)\n",
    "            #loss = tf.where(tf.equal(loss, -1e-1), tf.random_normal(loss.shape), loss)\n",
    "            #loss = tf.where(tf.equal(loss, 1e-1), tf.random_normal(loss.shape), loss)\n",
    "            \n",
    "            self.regularized_loss = loss\n",
    "            correct_prediction = tf.equal(tf.argmax(self.y_, 1), tf.argmax(self.y, 1))\n",
    "            self.tf_accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32), name = \"Accuracy\")\n",
    "\n",
    "        with tf.variable_scope(\"Optimizer\"):\n",
    "            learning_rate=self.lr\n",
    "            optimizer = tf.train.AdamOptimizer(learning_rate)\n",
    "            gradients, variables = zip(*optimizer.compute_gradients(self.regularized_loss))\n",
    "            gradients = [\n",
    "                None if gradient is None else tf.clip_by_value(gradient, -1, 1)\n",
    "                for gradient in gradients]\n",
    "            self.train_op = optimizer.apply_gradients(zip(gradients, variables))\n",
    "            #self.train_op = optimizer.minimize(self.regularized_loss)\n",
    "            \n",
    "        # add op for merging summary\n",
    "        #self.summary_op = tf.summary.merge_all()\n",
    "        self.pred = tf.argmax(self.y, axis = 1)\n",
    "        self.actual = tf.argmax(self.y_, axis = 1)\n",
    "\n",
    "        # add Saver ops\n",
    "        self.saver = tf.train.Saver()\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:08.915556Z",
     "start_time": "2017-07-24T00:03:08.631105Z"
    },
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "import time\n",
    "import sklearn.metrics as me \n",
    "\n",
    "class Train:    \n",
    "    \n",
    "    result = namedtuple(\"score\", ['epoch', 'no_of_features','hidden_layers','train_score', 'test_score', 'f1_score', 'test_score_20', 'f1_score_20', 'time_taken'])\n",
    "\n",
    "    predictions = {}\n",
    "    predictions_ = {}\n",
    "\n",
    "    results = []\n",
    "    \n",
    "    best_acc = 0\n",
    "    best_acc_global = 0\n",
    "    \n",
    "    def train(epochs, net, h,f, lrs):\n",
    "        batch_iterations = 200\n",
    "        train_loss = None\n",
    "        Train.best_acc = 0\n",
    "        os.makedirs(\"dataset/tf_dense_only_nsl_kdd-/hidden layers_{}_features count_{}\".format(epochs,h,f),\n",
    "                    exist_ok = True)\n",
    "        with tf.Session() as sess:\n",
    "            #summary_writer_train = tf.summary.FileWriter('./logs/kdd/VAE/training', graph=sess.graph)\n",
    "            #summary_writer_valid = tf.summary.FileWriter('./logs/kdd/VAE/validation')\n",
    "\n",
    "            sess.run(tf.global_variables_initializer())\n",
    "            start_time = time.perf_counter()\n",
    "            for c, lr in enumerate(lrs):\n",
    "                for epoch in range(1, (epochs+1)):\n",
    "                    x_train, x_valid, y_train, y_valid, = ms.train_test_split(preprocess.x_train, \n",
    "                                                                              preprocess.y_train, \n",
    "                                                                              test_size=0.1)\n",
    "                    batch_indices = np.array_split(np.arange(x_train.shape[0]), \n",
    "                                               batch_iterations)\n",
    "\n",
    "                    for i in batch_indices:\n",
    "\n",
    "                        def train_batch():\n",
    "                            nonlocal train_loss\n",
    "                            _, train_loss = sess.run([net.train_op, \n",
    "                                                               net.regularized_loss, \n",
    "                                                               ], #net.summary_op\n",
    "                                                              feed_dict={net.x: x_train[i,:], \n",
    "                                                                         net.y_: y_train[i,:], \n",
    "                                                                         net.keep_prob:0.5, net.lr:lr})\n",
    "\n",
    "                        train_batch()\n",
    "                        #summary_writer_train.add_summary(summary_str, epoch)\n",
    "                        while((train_loss > 1e4 or np.isnan(train_loss)) and epoch > 1):\n",
    "                            print(\"Step {} | Training Loss: {:.6f}\".format(epoch, train_loss))\n",
    "                            net.saver.restore(sess, \n",
    "                                              tf.train.latest_checkpoint('dataset/tf_dense_only_nsl_kdd-/hidden_layers_{}_features_count_{}'\n",
    "                                                                         .format(epochs,h,f)))\n",
    "                            train_batch()\n",
    "\n",
    "\n",
    "                    valid_accuracy = sess.run(net.tf_accuracy, #net.summary_op \n",
    "                                                          feed_dict={net.x: x_valid, \n",
    "                                                                     net.y_: y_valid, \n",
    "                                                                     net.keep_prob:1, net.lr:lr})\n",
    "                    #summary_writer_valid.add_summary(summary_str, epoch)\n",
    "\n",
    "\n",
    "                    accuracy, pred_value, actual_value, y_pred = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x: preprocess.x_test, \n",
    "                                                                             net.y_: preprocess.y_test, \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "                    f1_score = me.f1_score(actual_value, pred_value)\n",
    "                    accuracy_, pred_value_, actual_value_, y_pred_ = sess.run([net.tf_accuracy, \n",
    "                                                                   net.pred, \n",
    "                                                                   net.actual, net.y], \n",
    "                                                                  feed_dict={net.x: preprocess.x_test_, \n",
    "                                                                             net.y_: preprocess.y_test_, \n",
    "                                                                             net.keep_prob:1, net.lr:lr})\n",
    "                    f1_score_ = me.f1_score(actual_value_, pred_value_)\n",
    "                    \n",
    "                    print(\"Step {} | Training Loss: {:.6f} | Validation Accuracy: {:.6f}\".format(epoch, train_loss, valid_accuracy))\n",
    "                    print(\"Accuracy on Test data: {}, {}\".format(accuracy, accuracy_))\n",
    "\n",
    "                    if accuracy > Train.best_acc_global:\n",
    "                        Train.best_acc_global = accuracy\n",
    "                        Train.pred_value = pred_value\n",
    "                        Train.actual_value = actual_value\n",
    "                        Train.pred_value_ = pred_value_\n",
    "                        Train.actual_value_ = actual_value_\n",
    "                        Train.best_parameters = \"Hidden Layers:{}, Features Count:{}\".format(h, f)\n",
    "\n",
    "                    if accuracy > Train.best_acc:\n",
    "                        Train.best_acc = accuracy\n",
    "\n",
    "                        if not (np.isnan(train_loss)):\n",
    "                            net.saver.save(sess, \n",
    "                                       \"dataset/tf_dense_only_nsl_kdd-/hidden_layers_{}_features_count_{}\".format(h,f),\n",
    "                                        global_step = epochs)\n",
    "                        curr_pred = pd.DataFrame({\"Attack_prob\":y_pred[:,-2], \"Normal_prob\":y_pred[:, -1], \"Prediction\":pred_value, \"Actual\":actual_value})\n",
    "                        curr_pred_ = pd.DataFrame({\"Attack_prob\":y_pred_[:,-2], \"Normal_prob\":y_pred_[:, -1], \"Prediction\":pred_value_, \"Actual\": actual_value_})\n",
    "                        \n",
    "                        Train.predictions.update({\"{}_{}_{}\".format((epoch+1)*(c+1),f,h):(curr_pred, \n",
    "                                                   Train.result((epoch+1)*(c+1), f, h, valid_accuracy, accuracy, f1_score, accuracy_, f1_score_, time.perf_counter() - start_time))})\n",
    "                        Train.predictions_.update({\"{}_{}_{}\".format((epoch+1)*(c+1),f,h):(curr_pred_, \n",
    "                                                   Train.result((epoch+1)*(c+1), f, h, valid_accuracy, accuracy, f1_score, accuracy_, f1_score_, time.perf_counter() - start_time))})\n",
    "\n",
    "                        #Train.results.append(Train.result(epochs, f, h,valid_accuracy, accuracy))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:03:09.007865Z",
     "start_time": "2017-07-24T00:03:08.916979Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import itertools\n",
    "\n",
    "df_results = []\n",
    "past_scores = []\n",
    "\n",
    "class Hyperparameters:\n",
    "#    features_arr = [2, 4, 8, 16, 32, 64, 128, 256]\n",
    "#    hidden_layers_arr = [2, 4, 6, 10]\n",
    "\n",
    "    def start_training():\n",
    "        print(\"********************************** Training ******************************\")\n",
    "\n",
    "        global df_results\n",
    "        global past_scores\n",
    "        Train.predictions = {}\n",
    "        Train.predictions_ = {}\n",
    "\n",
    "        Train.results = []\n",
    "    \n",
    "        \n",
    "        features_arr = [1, 12, 24, 48, 122]\n",
    "        hidden_layers_arr = [1, 3]\n",
    "\n",
    "        epochs = [5]\n",
    "        lrs = [1e-5, 1e-6]\n",
    "        print(\"********************************** Entering Loop ******************************\")\n",
    "\n",
    "        for e, h, f in itertools.product(epochs, hidden_layers_arr, features_arr):\n",
    "            print(\"Current Layer Attributes - epochs:{} hidden layers:{} features count:{}\".format(e,h,f))\n",
    "            n = network(2,h,f)\n",
    "            n.build_layers()\n",
    "            Train.train(e, n, h,f, lrs)\n",
    "            \n",
    "        dict1 = {}\n",
    "        dict1_ = {}\n",
    "        dict2 = []\n",
    "\n",
    "        for k, (v1, v2) in Train.predictions.items():\n",
    "            dict1.update({k: v1})\n",
    "            dict2.append(v2)\n",
    "\n",
    "        for k, (v1_, v2) in Train.predictions_.items():\n",
    "            dict1_.update({k: v1_})\n",
    "\n",
    "        Train.predictions = dict1\n",
    "        Train.predictions_ = dict1_\n",
    "        \n",
    "        Train.results = dict2\n",
    "        df_results = pd.DataFrame(Train.results)\n",
    "\n",
    "        #temp = df_results.set_index(['no_of_features', 'hidden_layers'])\n",
    "\n",
    "        if not os.path.isfile('dataset/scores/tf_dense_only_nsl_kdd_scores_all-.pkl'):\n",
    "            past_scores = df_results\n",
    "        else:\n",
    "            past_scores = pd.read_pickle(\"dataset/scores/tf_dense_only_nsl_kdd_scores_all-.pkl\")\n",
    "            past_scores = past_scores.append(df_results, ignore_index=True)\n",
    "        past_scores.to_pickle(\"dataset/scores/tf_dense_only_nsl_kdd_scores_all-.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:20.470993Z",
     "start_time": "2017-07-24T00:03:09.009270Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "********************************** Training ******************************\n",
      "********************************** Entering Loop ******************************\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:1\n",
      "Step 1 | Training Loss: 0.804066 | Validation Accuracy: 0.543254\n",
      "Accuracy on Test data: 0.4374556541442871, 0.19578059017658234\n",
      "Step 2 | Training Loss: 0.709505 | Validation Accuracy: 0.588889\n",
      "Accuracy on Test data: 0.4477466344833374, 0.21392405033111572\n",
      "Step 3 | Training Loss: 0.660634 | Validation Accuracy: 0.601190\n",
      "Accuracy on Test data: 0.4582594037055969, 0.23265822231769562\n",
      "Step 4 | Training Loss: 0.602664 | Validation Accuracy: 0.730952\n",
      "Accuracy on Test data: 0.507762610912323, 0.24793249368667603\n",
      "Step 5 | Training Loss: 0.588001 | Validation Accuracy: 0.759524\n",
      "Accuracy on Test data: 0.5309173464775085, 0.2886075973510742\n",
      "Step 1 | Training Loss: 0.599571 | Validation Accuracy: 0.757143\n",
      "Accuracy on Test data: 0.5338005423545837, 0.29375526309013367\n",
      "Step 2 | Training Loss: 0.573258 | Validation Accuracy: 0.767857\n",
      "Accuracy on Test data: 0.5369499921798706, 0.299156129360199\n",
      "Step 3 | Training Loss: 0.576401 | Validation Accuracy: 0.754365\n",
      "Accuracy on Test data: 0.53974449634552, 0.30362868309020996\n",
      "Step 4 | Training Loss: 0.601985 | Validation Accuracy: 0.775794\n",
      "Accuracy on Test data: 0.5444464087486267, 0.31147679686546326\n",
      "Step 5 | Training Loss: 0.593091 | Validation Accuracy: 0.776190\n",
      "Accuracy on Test data: 0.5603264570236206, 0.3157805800437927\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:12\n",
      "Step 1 | Training Loss: 0.651385 | Validation Accuracy: 0.639286\n",
      "Accuracy on Test data: 0.591288149356842, 0.39518988132476807\n",
      "Step 2 | Training Loss: 0.626038 | Validation Accuracy: 0.763889\n",
      "Accuracy on Test data: 0.6734829545021057, 0.4113923907279968\n",
      "Step 3 | Training Loss: 0.621024 | Validation Accuracy: 0.815079\n",
      "Accuracy on Test data: 0.6867902874946594, 0.42405062913894653\n",
      "Step 4 | Training Loss: 0.604952 | Validation Accuracy: 0.845635\n",
      "Accuracy on Test data: 0.7069730162620544, 0.46025317907333374\n",
      "Step 5 | Training Loss: 0.533078 | Validation Accuracy: 0.860317\n",
      "Accuracy on Test data: 0.7197480201721191, 0.48320674896240234\n",
      "Step 1 | Training Loss: 0.539315 | Validation Accuracy: 0.868651\n",
      "Accuracy on Test data: 0.7216110825538635, 0.4866666793823242\n",
      "Step 2 | Training Loss: 0.540589 | Validation Accuracy: 0.865873\n",
      "Accuracy on Test data: 0.7237402200698853, 0.4903797507286072\n",
      "Step 3 | Training Loss: 0.527431 | Validation Accuracy: 0.878571\n",
      "Accuracy on Test data: 0.7267565727233887, 0.4957805871963501\n",
      "Step 4 | Training Loss: 0.531291 | Validation Accuracy: 0.888492\n",
      "Accuracy on Test data: 0.7296398282051086, 0.5010126829147339\n",
      "Step 5 | Training Loss: 0.525746 | Validation Accuracy: 0.888095\n",
      "Accuracy on Test data: 0.7312366962432861, 0.503628671169281\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:24\n",
      "Step 1 | Training Loss: 0.767678 | Validation Accuracy: 0.288889\n",
      "Accuracy on Test data: 0.43701207637786865, 0.6046413779258728\n",
      "Step 2 | Training Loss: 0.683623 | Validation Accuracy: 0.555159\n",
      "Accuracy on Test data: 0.6864797472953796, 0.6579746603965759\n",
      "Step 3 | Training Loss: 0.642832 | Validation Accuracy: 0.778175\n",
      "Accuracy on Test data: 0.8164921998977661, 0.6801687479019165\n",
      "Step 4 | Training Loss: 0.579741 | Validation Accuracy: 0.846429\n",
      "Accuracy on Test data: 0.8341465592384338, 0.696793258190155\n",
      "Step 5 | Training Loss: 0.553790 | Validation Accuracy: 0.871825\n",
      "Accuracy on Test data: 0.8423970937728882, 0.7072573900222778\n",
      "Step 1 | Training Loss: 0.572720 | Validation Accuracy: 0.869841\n",
      "Accuracy on Test data: 0.8425745368003845, 0.7075105309486389\n",
      "Step 2 | Training Loss: 0.573432 | Validation Accuracy: 0.879762\n",
      "Accuracy on Test data: 0.8437721729278564, 0.7093670964241028\n",
      "Step 3 | Training Loss: 0.557021 | Validation Accuracy: 0.885317\n",
      "Accuracy on Test data: 0.8442157506942749, 0.7102109789848328\n",
      "Step 4 | Training Loss: 0.558985 | Validation Accuracy: 0.884127\n",
      "Accuracy on Test data: 0.845014214515686, 0.7113924026489258\n",
      "Step 5 | Training Loss: 0.557387 | Validation Accuracy: 0.890873\n",
      "Accuracy on Test data: 0.8449698090553284, 0.7108860611915588\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:48\n",
      "Step 1 | Training Loss: 0.597744 | Validation Accuracy: 0.846825\n",
      "Accuracy on Test data: 0.8561035990715027, 0.750379741191864\n",
      "Step 2 | Training Loss: 0.547499 | Validation Accuracy: 0.864683\n",
      "Accuracy on Test data: 0.8491394519805908, 0.7291139364242554\n",
      "Step 3 | Training Loss: 0.538729 | Validation Accuracy: 0.898413\n",
      "Accuracy on Test data: 0.8504258394241333, 0.7295358777046204\n",
      "Step 4 | Training Loss: 0.485779 | Validation Accuracy: 0.907937\n",
      "Accuracy on Test data: 0.8552164435386658, 0.7378059029579163\n",
      "Step 5 | Training Loss: 0.482663 | Validation Accuracy: 0.916667\n",
      "Accuracy on Test data: 0.857123851776123, 0.7406750917434692\n",
      "Step 1 | Training Loss: 0.466812 | Validation Accuracy: 0.911905\n",
      "Accuracy on Test data: 0.857123851776123, 0.7406750917434692\n",
      "Step 2 | Training Loss: 0.495983 | Validation Accuracy: 0.912302\n",
      "Accuracy on Test data: 0.8573456406593323, 0.7408438920974731\n",
      "Step 3 | Training Loss: 0.492276 | Validation Accuracy: 0.915476\n",
      "Accuracy on Test data: 0.8575230836868286, 0.7411814332008362\n",
      "Step 4 | Training Loss: 0.505915 | Validation Accuracy: 0.914286\n",
      "Accuracy on Test data: 0.8578779101371765, 0.7418565154075623\n",
      "Step 5 | Training Loss: 0.484171 | Validation Accuracy: 0.917063\n",
      "Accuracy on Test data: 0.8577448725700378, 0.7415189743041992\n",
      "Current Layer Attributes - epochs:5 hidden layers:1 features count:122\n",
      "Step 1 | Training Loss: 0.752569 | Validation Accuracy: 0.426984\n",
      "Accuracy on Test data: 0.4384758770465851, 0.4375527501106262\n",
      "Step 2 | Training Loss: 0.665333 | Validation Accuracy: 0.583730\n",
      "Accuracy on Test data: 0.5985628366470337, 0.4886919856071472\n",
      "Step 3 | Training Loss: 0.620566 | Validation Accuracy: 0.662302\n",
      "Accuracy on Test data: 0.6890525221824646, 0.5228691697120667\n",
      "Step 4 | Training Loss: 0.589624 | Validation Accuracy: 0.823810\n",
      "Accuracy on Test data: 0.7759048938751221, 0.5993248820304871\n",
      "Step 5 | Training Loss: 0.557219 | Validation Accuracy: 0.869841\n",
      "Accuracy on Test data: 0.7939584851264954, 0.6238818764686584\n",
      "Step 1 | Training Loss: 0.594878 | Validation Accuracy: 0.874206\n",
      "Accuracy on Test data: 0.7950674295425415, 0.6243038177490234\n",
      "Step 2 | Training Loss: 0.531732 | Validation Accuracy: 0.882143\n",
      "Accuracy on Test data: 0.796664297580719, 0.6259071826934814\n",
      "Step 3 | Training Loss: 0.560829 | Validation Accuracy: 0.878175\n",
      "Accuracy on Test data: 0.7975957989692688, 0.6254852414131165\n",
      "Step 4 | Training Loss: 0.570878 | Validation Accuracy: 0.892063\n",
      "Accuracy on Test data: 0.798660397529602, 0.6264978647232056\n",
      "Step 5 | Training Loss: 0.561398 | Validation Accuracy: 0.884921\n",
      "Accuracy on Test data: 0.7987934947013855, 0.6267510652542114\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ritesh_malaiya/anaconda3/envs/p3/lib/python3.6/site-packages/sklearn/metrics/classification.py:1113: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 due to no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1 | Training Loss: 0.693112 | Validation Accuracy: 0.551190\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 2 | Training Loss: 0.693037 | Validation Accuracy: 0.546429\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 0.693286 | Validation Accuracy: 0.543254\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 0.693048 | Validation Accuracy: 0.530159\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.693126 | Validation Accuracy: 0.535714\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 1 | Training Loss: 0.692970 | Validation Accuracy: 0.531746\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 2 | Training Loss: 0.692914 | Validation Accuracy: 0.547619\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 3 | Training Loss: 0.692909 | Validation Accuracy: 0.523413\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 4 | Training Loss: 0.693070 | Validation Accuracy: 0.511905\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Step 5 | Training Loss: 0.693404 | Validation Accuracy: 0.534921\n",
      "Accuracy on Test data: 0.43075764179229736, 0.18160337209701538\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:12\n",
      "Step 1 | Training Loss: 0.691103 | Validation Accuracy: 0.700397\n",
      "Accuracy on Test data: 0.6388839483261108, 0.425907164812088\n",
      "Step 2 | Training Loss: 0.655879 | Validation Accuracy: 0.782143\n",
      "Accuracy on Test data: 0.6881210207939148, 0.43932488560676575\n",
      "Step 3 | Training Loss: 0.695034 | Validation Accuracy: 0.816667\n",
      "Accuracy on Test data: 0.7005411386489868, 0.45358648896217346\n",
      "Step 4 | Training Loss: 0.677474 | Validation Accuracy: 0.846429\n",
      "Accuracy on Test data: 0.7159332633018494, 0.47915610671043396\n",
      "Step 5 | Training Loss: 0.651454 | Validation Accuracy: 0.848413\n",
      "Accuracy on Test data: 0.7221433520317078, 0.48877638578414917\n",
      "Step 1 | Training Loss: 0.633732 | Validation Accuracy: 0.846825\n",
      "Accuracy on Test data: 0.7227200269699097, 0.4897046387195587\n",
      "Step 2 | Training Loss: 0.678854 | Validation Accuracy: 0.860317\n",
      "Accuracy on Test data: 0.7232966423034668, 0.4907172918319702\n",
      "Step 3 | Training Loss: 0.624039 | Validation Accuracy: 0.858333\n",
      "Accuracy on Test data: 0.7239620089530945, 0.49181434512138367\n",
      "Step 4 | Training Loss: 0.651784 | Validation Accuracy: 0.854365\n",
      "Accuracy on Test data: 0.7269783616065979, 0.4973839521408081\n",
      "Step 5 | Training Loss: 0.652090 | Validation Accuracy: 0.857540\n",
      "Accuracy on Test data: 0.7295954823493958, 0.502109706401825\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:24\n",
      "Step 1 | Training Loss: 0.730834 | Validation Accuracy: 0.349603\n",
      "Accuracy on Test data: 0.20045244693756104, 0.2586497962474823\n",
      "Step 2 | Training Loss: 0.712260 | Validation Accuracy: 0.574603\n",
      "Accuracy on Test data: 0.44016146659851074, 0.24329113960266113\n",
      "Step 3 | Training Loss: 0.688727 | Validation Accuracy: 0.670635\n",
      "Accuracy on Test data: 0.5702182650566101, 0.2827848196029663\n",
      "Step 4 | Training Loss: 0.637173 | Validation Accuracy: 0.773413\n",
      "Accuracy on Test data: 0.6251330971717834, 0.3086076080799103\n",
      "Step 5 | Training Loss: 0.679531 | Validation Accuracy: 0.807936\n",
      "Accuracy on Test data: 0.6705996990203857, 0.3924894630908966\n",
      "Step 1 | Training Loss: 0.655700 | Validation Accuracy: 0.816667\n",
      "Accuracy on Test data: 0.6737490892410278, 0.39822784066200256\n",
      "Step 2 | Training Loss: 0.658028 | Validation Accuracy: 0.815873\n",
      "Accuracy on Test data: 0.6775638461112976, 0.4049789011478424\n",
      "Step 3 | Training Loss: 0.640675 | Validation Accuracy: 0.821825\n",
      "Accuracy on Test data: 0.680890679359436, 0.4112236201763153\n",
      "Step 4 | Training Loss: 0.691339 | Validation Accuracy: 0.815079\n",
      "Accuracy on Test data: 0.6831085681915283, 0.4152742624282837\n",
      "Step 5 | Training Loss: 0.669821 | Validation Accuracy: 0.817460\n",
      "Accuracy on Test data: 0.6852821111679077, 0.41940927505493164\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:48\n",
      "Step 1 | Training Loss: 0.719790 | Validation Accuracy: 0.662302\n",
      "Accuracy on Test data: 0.49795955419540405, 0.2288607656955719\n",
      "Step 2 | Training Loss: 0.712641 | Validation Accuracy: 0.691270\n",
      "Accuracy on Test data: 0.5543381571769714, 0.2574683427810669\n",
      "Step 3 | Training Loss: 0.670495 | Validation Accuracy: 0.776190\n",
      "Accuracy on Test data: 0.6291696429252625, 0.3164556920528412\n",
      "Step 4 | Training Loss: 0.641001 | Validation Accuracy: 0.830159\n",
      "Accuracy on Test data: 0.6670510768890381, 0.3821941018104553\n",
      "Step 5 | Training Loss: 0.634173 | Validation Accuracy: 0.869444\n",
      "Accuracy on Test data: 0.6928229331970215, 0.4279325008392334\n",
      "Step 1 | Training Loss: 0.625137 | Validation Accuracy: 0.874603\n",
      "Accuracy on Test data: 0.6949964761734009, 0.4319831132888794\n",
      "Step 2 | Training Loss: 0.603623 | Validation Accuracy: 0.878571\n",
      "Accuracy on Test data: 0.6970812678337097, 0.4356118142604828\n",
      "Step 3 | Training Loss: 0.558584 | Validation Accuracy: 0.884127\n",
      "Accuracy on Test data: 0.7000532150268555, 0.44092828035354614\n",
      "Step 4 | Training Loss: 0.620057 | Validation Accuracy: 0.879762\n",
      "Accuracy on Test data: 0.7027590274810791, 0.445991575717926\n",
      "Step 5 | Training Loss: 0.575750 | Validation Accuracy: 0.887698\n",
      "Accuracy on Test data: 0.7040454149246216, 0.44835442304611206\n",
      "Current Layer Attributes - epochs:5 hidden layers:3 features count:122\n",
      "Step 1 | Training Loss: 0.668684 | Validation Accuracy: 0.768254\n",
      "Accuracy on Test data: 0.7576738595962524, 0.5873417854309082\n",
      "Step 2 | Training Loss: 0.641573 | Validation Accuracy: 0.809921\n",
      "Accuracy on Test data: 0.7869943380355835, 0.6220253109931946\n",
      "Step 3 | Training Loss: 0.642092 | Validation Accuracy: 0.875397\n",
      "Accuracy on Test data: 0.8244765996932983, 0.6774683594703674\n",
      "Step 4 | Training Loss: 0.625123 | Validation Accuracy: 0.874603\n",
      "Accuracy on Test data: 0.8274485468864441, 0.6827847957611084\n",
      "Step 5 | Training Loss: 0.566429 | Validation Accuracy: 0.894841\n",
      "Accuracy on Test data: 0.8262509107589722, 0.6793248653411865\n",
      "Step 1 | Training Loss: 0.622355 | Validation Accuracy: 0.895635\n",
      "Accuracy on Test data: 0.8262065052986145, 0.6790717244148254\n",
      "Step 2 | Training Loss: 0.568567 | Validation Accuracy: 0.894048\n",
      "Accuracy on Test data: 0.8259403705596924, 0.6783966422080994\n",
      "Step 3 | Training Loss: 0.596171 | Validation Accuracy: 0.900794\n",
      "Accuracy on Test data: 0.8257185816764832, 0.6779747009277344\n",
      "Step 4 | Training Loss: 0.595517 | Validation Accuracy: 0.892460\n",
      "Accuracy on Test data: 0.825762927532196, 0.6778903007507324\n",
      "Step 5 | Training Loss: 0.553253 | Validation Accuracy: 0.899206\n",
      "Accuracy on Test data: 0.8259847164154053, 0.6780591011047363\n"
     ]
    }
   ],
   "source": [
    "#%%timeit -r 10\n",
    "#capture\n",
    "Hyperparameters.start_training()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-06-18T20:12:01.591604Z",
     "start_time": "2017-06-18T20:12:01.586451Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:20.475025Z",
     "start_time": "2017-07-24T00:04:20.472588Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#g = df_results.groupby(by=['no_of_features'])\n",
    "#idx = g['test_score'].transform(max) == df_results['test_score']\n",
    "#df_results[idx].sort_values(by = 'test_score', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:20.484963Z",
     "start_time": "2017-07-24T00:04:20.476309Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#g = df_results.groupby(by=['no_of_features'])\n",
    "#idx = g['test_score_20'].transform(max) == df_results['test_score_20']\n",
    "#df_results[idx].sort_values(by = 'test_score_20', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:20.490842Z",
     "start_time": "2017-07-24T00:04:20.486263Z"
    },
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "#df_results.sort_values(by = 'test_score', ascending = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:20.497890Z",
     "start_time": "2017-07-24T00:04:20.492179Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Train.predictions_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:20.790921Z",
     "start_time": "2017-07-24T00:04:20.499172Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pd.Panel(Train.predictions).to_pickle(\"dataset/tf_dense_only_nsl_kdd_predictions-.pkl\")\n",
    "pd.Panel(Train.predictions_).to_pickle(\"dataset/tf_dense_only_nsl_kdd_predictions-__.pkl\")\n",
    "\n",
    "df_results.to_pickle(\"dataset/tf_dense_only_nsl_kdd_scores-.pkl\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:20.864233Z",
     "start_time": "2017-07-24T00:04:20.792427Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "\n",
    "def plot_confusion_matrix(cm, classes,\n",
    "                          normalize=False,\n",
    "                          title='Confusion matrix',\n",
    "                          cmap=plt.cm.Blues):\n",
    "    \"\"\"\n",
    "    This function prints and plots the confusion matrix.\n",
    "    Normalization can be applied by setting `normalize=True`.\n",
    "    \"\"\"\n",
    "    np.set_printoptions(precision=4)\n",
    "\n",
    "    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n",
    "    plt.title(title)\n",
    "    plt.colorbar()\n",
    "    tick_marks = np.arange(len(classes))\n",
    "    plt.xticks(tick_marks, classes, rotation=45)\n",
    "    plt.yticks(tick_marks, classes)\n",
    "\n",
    "    if normalize:\n",
    "        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n",
    "        #print(\"Normalized confusion matrix\")\n",
    "    else:\n",
    "        #print('Confusion matrix, without normalization')\n",
    "        pass\n",
    "    \n",
    "    #print(cm)\n",
    "\n",
    "    label = [[\"\\n True Negative\", \"\\n False Positive \\n Type II Error\"],\n",
    "             [\"\\n False Negative \\n Type I Error\", \"\\n True Positive\"]\n",
    "            ]\n",
    "    \n",
    "    thresh = cm.max() / 2.\n",
    "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
    "        \n",
    "        plt.text(j, i, \"{} {}\".format(cm[i, j].round(4), label[i][j]),\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if cm[i, j] > thresh else \"black\")\n",
    "\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('True label')\n",
    "    plt.xlabel('Predicted label')\n",
    "\n",
    "def plot(actual_value, pred_value):\n",
    "    from sklearn.metrics import confusion_matrix\n",
    "\n",
    "    cm_2labels = confusion_matrix(y_pred = pred_value, y_true = actual_value)\n",
    "    plt.figure(figsize=[6,6])\n",
    "    plot_confusion_matrix(cm_2labels, ['Normal', 'Attack'], normalize = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.078539Z",
     "start_time": "2017-07-24T00:04:20.865690Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#plot(actual_value = Train.actual_value, pred_value = Train.pred_value)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.083221Z",
     "start_time": "2017-07-24T00:04:21.080021Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#plot(actual_value = Train.actual_value_, pred_value = Train.pred_value_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.088188Z",
     "start_time": "2017-07-24T00:04:21.084457Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "past_scores = pd.read_pickle(\"dataset/scores/tf_dense_only_nsl_kdd_scores_all-.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.123016Z",
     "start_time": "2017-07-24T00:04:21.089482Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>f1_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>1</td>\n",
       "      <td>0.846825</td>\n",
       "      <td>0.856104</td>\n",
       "      <td>0.869488</td>\n",
       "      <td>0.750380</td>\n",
       "      <td>0.839570</td>\n",
       "      <td>0.642131</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>10</td>\n",
       "      <td>48</td>\n",
       "      <td>1</td>\n",
       "      <td>0.914286</td>\n",
       "      <td>0.857878</td>\n",
       "      <td>0.867734</td>\n",
       "      <td>0.741857</td>\n",
       "      <td>0.828982</td>\n",
       "      <td>4.327987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>8</td>\n",
       "      <td>48</td>\n",
       "      <td>1</td>\n",
       "      <td>0.915476</td>\n",
       "      <td>0.857523</td>\n",
       "      <td>0.867393</td>\n",
       "      <td>0.741181</td>\n",
       "      <td>0.828516</td>\n",
       "      <td>3.768181</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>6</td>\n",
       "      <td>48</td>\n",
       "      <td>1</td>\n",
       "      <td>0.912302</td>\n",
       "      <td>0.857346</td>\n",
       "      <td>0.867195</td>\n",
       "      <td>0.740844</td>\n",
       "      <td>0.828234</td>\n",
       "      <td>3.158489</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>10</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.884127</td>\n",
       "      <td>0.845014</td>\n",
       "      <td>0.847968</td>\n",
       "      <td>0.711392</td>\n",
       "      <td>0.795185</td>\n",
       "      <td>5.346153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.885317</td>\n",
       "      <td>0.844216</td>\n",
       "      <td>0.847078</td>\n",
       "      <td>0.710211</td>\n",
       "      <td>0.794199</td>\n",
       "      <td>4.741357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.879762</td>\n",
       "      <td>0.843772</td>\n",
       "      <td>0.846576</td>\n",
       "      <td>0.709367</td>\n",
       "      <td>0.793476</td>\n",
       "      <td>4.194588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.869841</td>\n",
       "      <td>0.842575</td>\n",
       "      <td>0.845299</td>\n",
       "      <td>0.707511</td>\n",
       "      <td>0.791957</td>\n",
       "      <td>3.586408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.846429</td>\n",
       "      <td>0.834147</td>\n",
       "      <td>0.836632</td>\n",
       "      <td>0.696793</td>\n",
       "      <td>0.783385</td>\n",
       "      <td>2.446598</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>5</td>\n",
       "      <td>122</td>\n",
       "      <td>3</td>\n",
       "      <td>0.874603</td>\n",
       "      <td>0.827449</td>\n",
       "      <td>0.836445</td>\n",
       "      <td>0.682785</td>\n",
       "      <td>0.784028</td>\n",
       "      <td>3.972664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>4</td>\n",
       "      <td>122</td>\n",
       "      <td>3</td>\n",
       "      <td>0.875397</td>\n",
       "      <td>0.824477</td>\n",
       "      <td>0.834484</td>\n",
       "      <td>0.677468</td>\n",
       "      <td>0.781899</td>\n",
       "      <td>2.972374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>3</td>\n",
       "      <td>122</td>\n",
       "      <td>3</td>\n",
       "      <td>0.809921</td>\n",
       "      <td>0.786994</td>\n",
       "      <td>0.801472</td>\n",
       "      <td>0.622025</td>\n",
       "      <td>0.745786</td>\n",
       "      <td>1.957672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>12</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.884921</td>\n",
       "      <td>0.798793</td>\n",
       "      <td>0.788984</td>\n",
       "      <td>0.626751</td>\n",
       "      <td>0.710366</td>\n",
       "      <td>5.995009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>10</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.892063</td>\n",
       "      <td>0.798660</td>\n",
       "      <td>0.788795</td>\n",
       "      <td>0.626498</td>\n",
       "      <td>0.710075</td>\n",
       "      <td>5.441039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>8</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.878175</td>\n",
       "      <td>0.797596</td>\n",
       "      <td>0.787659</td>\n",
       "      <td>0.625485</td>\n",
       "      <td>0.709289</td>\n",
       "      <td>4.856589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>6</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.882143</td>\n",
       "      <td>0.796664</td>\n",
       "      <td>0.786592</td>\n",
       "      <td>0.625907</td>\n",
       "      <td>0.709749</td>\n",
       "      <td>4.193246</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>4</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.874206</td>\n",
       "      <td>0.795067</td>\n",
       "      <td>0.784696</td>\n",
       "      <td>0.624304</td>\n",
       "      <td>0.708219</td>\n",
       "      <td>3.609194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>2</td>\n",
       "      <td>122</td>\n",
       "      <td>3</td>\n",
       "      <td>0.768254</td>\n",
       "      <td>0.757674</td>\n",
       "      <td>0.771661</td>\n",
       "      <td>0.587342</td>\n",
       "      <td>0.714702</td>\n",
       "      <td>0.950984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>5</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.823810</td>\n",
       "      <td>0.775905</td>\n",
       "      <td>0.761900</td>\n",
       "      <td>0.599325</td>\n",
       "      <td>0.683129</td>\n",
       "      <td>2.403930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.857540</td>\n",
       "      <td>0.729595</td>\n",
       "      <td>0.704365</td>\n",
       "      <td>0.502110</td>\n",
       "      <td>0.587355</td>\n",
       "      <td>8.040761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.888095</td>\n",
       "      <td>0.731237</td>\n",
       "      <td>0.701277</td>\n",
       "      <td>0.503629</td>\n",
       "      <td>0.579437</td>\n",
       "      <td>5.775745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.854365</td>\n",
       "      <td>0.726978</td>\n",
       "      <td>0.700676</td>\n",
       "      <td>0.497384</td>\n",
       "      <td>0.581800</td>\n",
       "      <td>7.221911</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.555159</td>\n",
       "      <td>0.686480</td>\n",
       "      <td>0.699566</td>\n",
       "      <td>0.657975</td>\n",
       "      <td>0.751456</td>\n",
       "      <td>1.181485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>10</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.888492</td>\n",
       "      <td>0.729640</td>\n",
       "      <td>0.698879</td>\n",
       "      <td>0.501013</td>\n",
       "      <td>0.576099</td>\n",
       "      <td>5.240943</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.858333</td>\n",
       "      <td>0.723962</td>\n",
       "      <td>0.696454</td>\n",
       "      <td>0.491814</td>\n",
       "      <td>0.575317</td>\n",
       "      <td>6.397934</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.860317</td>\n",
       "      <td>0.723297</td>\n",
       "      <td>0.695589</td>\n",
       "      <td>0.490717</td>\n",
       "      <td>0.574130</td>\n",
       "      <td>5.594408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.846825</td>\n",
       "      <td>0.722720</td>\n",
       "      <td>0.694880</td>\n",
       "      <td>0.489705</td>\n",
       "      <td>0.573163</td>\n",
       "      <td>4.851391</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>8</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.878571</td>\n",
       "      <td>0.726757</td>\n",
       "      <td>0.694626</td>\n",
       "      <td>0.495781</td>\n",
       "      <td>0.569556</td>\n",
       "      <td>4.638695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>6</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.865873</td>\n",
       "      <td>0.723740</td>\n",
       "      <td>0.690088</td>\n",
       "      <td>0.490380</td>\n",
       "      <td>0.562677</td>\n",
       "      <td>4.093649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.868651</td>\n",
       "      <td>0.721611</td>\n",
       "      <td>0.686951</td>\n",
       "      <td>0.486667</td>\n",
       "      <td>0.558082</td>\n",
       "      <td>3.485513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.846429</td>\n",
       "      <td>0.715933</td>\n",
       "      <td>0.685709</td>\n",
       "      <td>0.479156</td>\n",
       "      <td>0.560899</td>\n",
       "      <td>3.209052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>5</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.845635</td>\n",
       "      <td>0.706973</td>\n",
       "      <td>0.664397</td>\n",
       "      <td>0.460253</td>\n",
       "      <td>0.523256</td>\n",
       "      <td>2.335920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>12</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.887698</td>\n",
       "      <td>0.704045</td>\n",
       "      <td>0.655265</td>\n",
       "      <td>0.448354</td>\n",
       "      <td>0.504960</td>\n",
       "      <td>8.769443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>10</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.879762</td>\n",
       "      <td>0.702759</td>\n",
       "      <td>0.653032</td>\n",
       "      <td>0.445992</td>\n",
       "      <td>0.501329</td>\n",
       "      <td>7.874758</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.782143</td>\n",
       "      <td>0.688121</td>\n",
       "      <td>0.649798</td>\n",
       "      <td>0.439325</td>\n",
       "      <td>0.514824</td>\n",
       "      <td>1.595154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>8</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.884127</td>\n",
       "      <td>0.700053</td>\n",
       "      <td>0.648727</td>\n",
       "      <td>0.440928</td>\n",
       "      <td>0.494391</td>\n",
       "      <td>6.982720</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>6</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.878571</td>\n",
       "      <td>0.697081</td>\n",
       "      <td>0.643859</td>\n",
       "      <td>0.435612</td>\n",
       "      <td>0.486802</td>\n",
       "      <td>6.084052</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>4</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.874603</td>\n",
       "      <td>0.694996</td>\n",
       "      <td>0.640414</td>\n",
       "      <td>0.431983</td>\n",
       "      <td>0.481633</td>\n",
       "      <td>5.269588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>12</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.817460</td>\n",
       "      <td>0.685282</td>\n",
       "      <td>0.622265</td>\n",
       "      <td>0.419409</td>\n",
       "      <td>0.456298</td>\n",
       "      <td>8.399888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.763889</td>\n",
       "      <td>0.673483</td>\n",
       "      <td>0.620547</td>\n",
       "      <td>0.411392</td>\n",
       "      <td>0.465476</td>\n",
       "      <td>1.176470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>10</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.815079</td>\n",
       "      <td>0.683109</td>\n",
       "      <td>0.618417</td>\n",
       "      <td>0.415274</td>\n",
       "      <td>0.449774</td>\n",
       "      <td>7.532192</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>8</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.821825</td>\n",
       "      <td>0.680891</td>\n",
       "      <td>0.614717</td>\n",
       "      <td>0.411224</td>\n",
       "      <td>0.443842</td>\n",
       "      <td>6.746385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>6</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.815873</td>\n",
       "      <td>0.677564</td>\n",
       "      <td>0.609088</td>\n",
       "      <td>0.404979</td>\n",
       "      <td>0.434518</td>\n",
       "      <td>5.869643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.816667</td>\n",
       "      <td>0.673749</td>\n",
       "      <td>0.602497</td>\n",
       "      <td>0.398228</td>\n",
       "      <td>0.424130</td>\n",
       "      <td>4.998380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>5</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.830159</td>\n",
       "      <td>0.667051</td>\n",
       "      <td>0.593083</td>\n",
       "      <td>0.382194</td>\n",
       "      <td>0.407254</td>\n",
       "      <td>3.501897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0.700397</td>\n",
       "      <td>0.638884</td>\n",
       "      <td>0.575658</td>\n",
       "      <td>0.425907</td>\n",
       "      <td>0.498858</td>\n",
       "      <td>0.850992</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>1</td>\n",
       "      <td>0.288889</td>\n",
       "      <td>0.437012</td>\n",
       "      <td>0.548552</td>\n",
       "      <td>0.604641</td>\n",
       "      <td>0.708263</td>\n",
       "      <td>0.624594</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>0.639286</td>\n",
       "      <td>0.591288</td>\n",
       "      <td>0.541729</td>\n",
       "      <td>0.395190</td>\n",
       "      <td>0.451855</td>\n",
       "      <td>0.631919</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>5</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.773413</td>\n",
       "      <td>0.625133</td>\n",
       "      <td>0.513612</td>\n",
       "      <td>0.308608</td>\n",
       "      <td>0.275020</td>\n",
       "      <td>3.390746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>3</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.583730</td>\n",
       "      <td>0.598563</td>\n",
       "      <td>0.505140</td>\n",
       "      <td>0.488692</td>\n",
       "      <td>0.570192</td>\n",
       "      <td>1.208710</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>12</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.776190</td>\n",
       "      <td>0.560326</td>\n",
       "      <td>0.377152</td>\n",
       "      <td>0.315781</td>\n",
       "      <td>0.288022</td>\n",
       "      <td>5.756575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>2</td>\n",
       "      <td>122</td>\n",
       "      <td>1</td>\n",
       "      <td>0.426984</td>\n",
       "      <td>0.438476</td>\n",
       "      <td>0.367208</td>\n",
       "      <td>0.437553</td>\n",
       "      <td>0.514955</td>\n",
       "      <td>0.576209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>3</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.691270</td>\n",
       "      <td>0.554338</td>\n",
       "      <td>0.363913</td>\n",
       "      <td>0.257468</td>\n",
       "      <td>0.182629</td>\n",
       "      <td>1.817415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.775794</td>\n",
       "      <td>0.544446</td>\n",
       "      <td>0.339295</td>\n",
       "      <td>0.311477</td>\n",
       "      <td>0.279686</td>\n",
       "      <td>5.161612</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.754365</td>\n",
       "      <td>0.539744</td>\n",
       "      <td>0.327631</td>\n",
       "      <td>0.303629</td>\n",
       "      <td>0.265052</td>\n",
       "      <td>4.625136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.767857</td>\n",
       "      <td>0.536950</td>\n",
       "      <td>0.320510</td>\n",
       "      <td>0.299156</td>\n",
       "      <td>0.256424</td>\n",
       "      <td>4.075766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.757143</td>\n",
       "      <td>0.533801</td>\n",
       "      <td>0.312533</td>\n",
       "      <td>0.293755</td>\n",
       "      <td>0.246104</td>\n",
       "      <td>3.478816</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.730952</td>\n",
       "      <td>0.507763</td>\n",
       "      <td>0.241335</td>\n",
       "      <td>0.247932</td>\n",
       "      <td>0.150753</td>\n",
       "      <td>2.334681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>2</td>\n",
       "      <td>48</td>\n",
       "      <td>3</td>\n",
       "      <td>0.662302</td>\n",
       "      <td>0.497960</td>\n",
       "      <td>0.212825</td>\n",
       "      <td>0.228861</td>\n",
       "      <td>0.111435</td>\n",
       "      <td>0.927484</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.574603</td>\n",
       "      <td>0.440161</td>\n",
       "      <td>0.121651</td>\n",
       "      <td>0.243291</td>\n",
       "      <td>0.151093</td>\n",
       "      <td>1.698722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>2</td>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0.349603</td>\n",
       "      <td>0.200452</td>\n",
       "      <td>0.121375</td>\n",
       "      <td>0.258650</td>\n",
       "      <td>0.218486</td>\n",
       "      <td>0.865629</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.588889</td>\n",
       "      <td>0.447747</td>\n",
       "      <td>0.061086</td>\n",
       "      <td>0.213924</td>\n",
       "      <td>0.076168</td>\n",
       "      <td>1.196932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.543254</td>\n",
       "      <td>0.437456</td>\n",
       "      <td>0.026857</td>\n",
       "      <td>0.195781</td>\n",
       "      <td>0.034448</td>\n",
       "      <td>0.650613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0.551190</td>\n",
       "      <td>0.430758</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.181603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.857066</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    epoch  no_of_features  hidden_layers  train_score  test_score  f1_score  \\\n",
       "23      2              48              1     0.846825    0.856104  0.869488   \n",
       "26     10              48              1     0.914286    0.857878  0.867734   \n",
       "25      8              48              1     0.915476    0.857523  0.867393   \n",
       "24      6              48              1     0.912302    0.857346  0.867195   \n",
       "22     10              24              1     0.884127    0.845014  0.847968   \n",
       "21      8              24              1     0.885317    0.844216  0.847078   \n",
       "20      6              24              1     0.879762    0.843772  0.846576   \n",
       "18      4              24              1     0.869841    0.842575  0.845299   \n",
       "19      5              24              1     0.846429    0.834147  0.836632   \n",
       "63      5             122              3     0.874603    0.827449  0.836445   \n",
       "62      4             122              3     0.875397    0.824477  0.834484   \n",
       "61      3             122              3     0.809921    0.786994  0.801472   \n",
       "34     12             122              1     0.884921    0.798793  0.788984   \n",
       "33     10             122              1     0.892063    0.798660  0.788795   \n",
       "32      8             122              1     0.878175    0.797596  0.787659   \n",
       "31      6             122              1     0.882143    0.796664  0.786592   \n",
       "29      4             122              1     0.874206    0.795067  0.784696   \n",
       "60      2             122              3     0.768254    0.757674  0.771661   \n",
       "30      5             122              1     0.823810    0.775905  0.761900   \n",
       "43     12              12              3     0.857540    0.729595  0.704365   \n",
       "15     12              12              1     0.888095    0.731237  0.701277   \n",
       "42     10              12              3     0.854365    0.726978  0.700676   \n",
       "17      3              24              1     0.555159    0.686480  0.699566   \n",
       "14     10              12              1     0.888492    0.729640  0.698879   \n",
       "41      8              12              3     0.858333    0.723962  0.696454   \n",
       "40      6              12              3     0.860317    0.723297  0.695589   \n",
       "38      4              12              3     0.846825    0.722720  0.694880   \n",
       "13      8              12              1     0.878571    0.726757  0.694626   \n",
       "12      6              12              1     0.865873    0.723740  0.690088   \n",
       "10      4              12              1     0.868651    0.721611  0.686951   \n",
       "39      5              12              3     0.846429    0.715933  0.685709   \n",
       "11      5              12              1     0.845635    0.706973  0.664397   \n",
       "59     12              48              3     0.887698    0.704045  0.655265   \n",
       "58     10              48              3     0.879762    0.702759  0.653032   \n",
       "37      3              12              3     0.782143    0.688121  0.649798   \n",
       "57      8              48              3     0.884127    0.700053  0.648727   \n",
       "56      6              48              3     0.878571    0.697081  0.643859   \n",
       "54      4              48              3     0.874603    0.694996  0.640414   \n",
       "51     12              24              3     0.817460    0.685282  0.622265   \n",
       "9       3              12              1     0.763889    0.673483  0.620547   \n",
       "50     10              24              3     0.815079    0.683109  0.618417   \n",
       "49      8              24              3     0.821825    0.680891  0.614717   \n",
       "48      6              24              3     0.815873    0.677564  0.609088   \n",
       "46      4              24              3     0.816667    0.673749  0.602497   \n",
       "55      5              48              3     0.830159    0.667051  0.593083   \n",
       "36      2              12              3     0.700397    0.638884  0.575658   \n",
       "16      2              24              1     0.288889    0.437012  0.548552   \n",
       "8       2              12              1     0.639286    0.591288  0.541729   \n",
       "47      5              24              3     0.773413    0.625133  0.513612   \n",
       "28      3             122              1     0.583730    0.598563  0.505140   \n",
       "7      12               1              1     0.776190    0.560326  0.377152   \n",
       "27      2             122              1     0.426984    0.438476  0.367208   \n",
       "53      3              48              3     0.691270    0.554338  0.363913   \n",
       "6      10               1              1     0.775794    0.544446  0.339295   \n",
       "5       8               1              1     0.754365    0.539744  0.327631   \n",
       "4       6               1              1     0.767857    0.536950  0.320510   \n",
       "2       4               1              1     0.757143    0.533801  0.312533   \n",
       "3       5               1              1     0.730952    0.507763  0.241335   \n",
       "52      2              48              3     0.662302    0.497960  0.212825   \n",
       "45      3              24              3     0.574603    0.440161  0.121651   \n",
       "44      2              24              3     0.349603    0.200452  0.121375   \n",
       "1       3               1              1     0.588889    0.447747  0.061086   \n",
       "0       2               1              1     0.543254    0.437456  0.026857   \n",
       "35      2               1              3     0.551190    0.430758  0.000000   \n",
       "\n",
       "    test_score_20  f1_score_20  time_taken  \n",
       "23       0.750380     0.839570    0.642131  \n",
       "26       0.741857     0.828982    4.327987  \n",
       "25       0.741181     0.828516    3.768181  \n",
       "24       0.740844     0.828234    3.158489  \n",
       "22       0.711392     0.795185    5.346153  \n",
       "21       0.710211     0.794199    4.741357  \n",
       "20       0.709367     0.793476    4.194588  \n",
       "18       0.707511     0.791957    3.586408  \n",
       "19       0.696793     0.783385    2.446598  \n",
       "63       0.682785     0.784028    3.972664  \n",
       "62       0.677468     0.781899    2.972374  \n",
       "61       0.622025     0.745786    1.957672  \n",
       "34       0.626751     0.710366    5.995009  \n",
       "33       0.626498     0.710075    5.441039  \n",
       "32       0.625485     0.709289    4.856589  \n",
       "31       0.625907     0.709749    4.193246  \n",
       "29       0.624304     0.708219    3.609194  \n",
       "60       0.587342     0.714702    0.950984  \n",
       "30       0.599325     0.683129    2.403930  \n",
       "43       0.502110     0.587355    8.040761  \n",
       "15       0.503629     0.579437    5.775745  \n",
       "42       0.497384     0.581800    7.221911  \n",
       "17       0.657975     0.751456    1.181485  \n",
       "14       0.501013     0.576099    5.240943  \n",
       "41       0.491814     0.575317    6.397934  \n",
       "40       0.490717     0.574130    5.594408  \n",
       "38       0.489705     0.573163    4.851391  \n",
       "13       0.495781     0.569556    4.638695  \n",
       "12       0.490380     0.562677    4.093649  \n",
       "10       0.486667     0.558082    3.485513  \n",
       "39       0.479156     0.560899    3.209052  \n",
       "11       0.460253     0.523256    2.335920  \n",
       "59       0.448354     0.504960    8.769443  \n",
       "58       0.445992     0.501329    7.874758  \n",
       "37       0.439325     0.514824    1.595154  \n",
       "57       0.440928     0.494391    6.982720  \n",
       "56       0.435612     0.486802    6.084052  \n",
       "54       0.431983     0.481633    5.269588  \n",
       "51       0.419409     0.456298    8.399888  \n",
       "9        0.411392     0.465476    1.176470  \n",
       "50       0.415274     0.449774    7.532192  \n",
       "49       0.411224     0.443842    6.746385  \n",
       "48       0.404979     0.434518    5.869643  \n",
       "46       0.398228     0.424130    4.998380  \n",
       "55       0.382194     0.407254    3.501897  \n",
       "36       0.425907     0.498858    0.850992  \n",
       "16       0.604641     0.708263    0.624594  \n",
       "8        0.395190     0.451855    0.631919  \n",
       "47       0.308608     0.275020    3.390746  \n",
       "28       0.488692     0.570192    1.208710  \n",
       "7        0.315781     0.288022    5.756575  \n",
       "27       0.437553     0.514955    0.576209  \n",
       "53       0.257468     0.182629    1.817415  \n",
       "6        0.311477     0.279686    5.161612  \n",
       "5        0.303629     0.265052    4.625136  \n",
       "4        0.299156     0.256424    4.075766  \n",
       "2        0.293755     0.246104    3.478816  \n",
       "3        0.247932     0.150753    2.334681  \n",
       "52       0.228861     0.111435    0.927484  \n",
       "45       0.243291     0.151093    1.698722  \n",
       "44       0.258650     0.218486    0.865629  \n",
       "1        0.213924     0.076168    1.196932  \n",
       "0        0.195781     0.034448    0.650613  \n",
       "35       0.181603     0.000000    0.857066  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "past_scores.sort_values(by='f1_score',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.144849Z",
     "start_time": "2017-07-24T00:04:21.124294Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>f1_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0.914286</td>\n",
       "      <td>0.857878</td>\n",
       "      <td>0.867734</td>\n",
       "      <td>0.741857</td>\n",
       "      <td>0.828982</td>\n",
       "      <td>4.327987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <th>1</th>\n",
       "      <td>10</td>\n",
       "      <td>0.884127</td>\n",
       "      <td>0.845014</td>\n",
       "      <td>0.847968</td>\n",
       "      <td>0.711392</td>\n",
       "      <td>0.795185</td>\n",
       "      <td>5.346153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">122</th>\n",
       "      <th>3</th>\n",
       "      <td>5</td>\n",
       "      <td>0.874603</td>\n",
       "      <td>0.827449</td>\n",
       "      <td>0.836445</td>\n",
       "      <td>0.682785</td>\n",
       "      <td>0.784028</td>\n",
       "      <td>3.972664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>0.884921</td>\n",
       "      <td>0.798793</td>\n",
       "      <td>0.788984</td>\n",
       "      <td>0.626751</td>\n",
       "      <td>0.710366</td>\n",
       "      <td>5.995009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">12</th>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>0.888095</td>\n",
       "      <td>0.731237</td>\n",
       "      <td>0.701277</td>\n",
       "      <td>0.503629</td>\n",
       "      <td>0.579437</td>\n",
       "      <td>5.775745</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>0.857540</td>\n",
       "      <td>0.729595</td>\n",
       "      <td>0.704365</td>\n",
       "      <td>0.502110</td>\n",
       "      <td>0.587355</td>\n",
       "      <td>8.040761</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>0.887698</td>\n",
       "      <td>0.704045</td>\n",
       "      <td>0.655265</td>\n",
       "      <td>0.448354</td>\n",
       "      <td>0.504960</td>\n",
       "      <td>8.769443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <th>3</th>\n",
       "      <td>12</td>\n",
       "      <td>0.817460</td>\n",
       "      <td>0.685282</td>\n",
       "      <td>0.622265</td>\n",
       "      <td>0.419409</td>\n",
       "      <td>0.456298</td>\n",
       "      <td>8.399888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>12</td>\n",
       "      <td>0.776190</td>\n",
       "      <td>0.560326</td>\n",
       "      <td>0.377152</td>\n",
       "      <td>0.315781</td>\n",
       "      <td>0.288022</td>\n",
       "      <td>5.756575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2</td>\n",
       "      <td>0.551190</td>\n",
       "      <td>0.430758</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.181603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.857066</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              epoch  train_score  test_score  f1_score  \\\n",
       "no_of_features hidden_layers                                             \n",
       "48             1                 10     0.914286    0.857878  0.867734   \n",
       "24             1                 10     0.884127    0.845014  0.847968   \n",
       "122            3                  5     0.874603    0.827449  0.836445   \n",
       "               1                 12     0.884921    0.798793  0.788984   \n",
       "12             1                 12     0.888095    0.731237  0.701277   \n",
       "               3                 12     0.857540    0.729595  0.704365   \n",
       "48             3                 12     0.887698    0.704045  0.655265   \n",
       "24             3                 12     0.817460    0.685282  0.622265   \n",
       "1              1                 12     0.776190    0.560326  0.377152   \n",
       "               3                  2     0.551190    0.430758  0.000000   \n",
       "\n",
       "                              test_score_20  f1_score_20  time_taken  \n",
       "no_of_features hidden_layers                                          \n",
       "48             1                   0.741857     0.828982    4.327987  \n",
       "24             1                   0.711392     0.795185    5.346153  \n",
       "122            3                   0.682785     0.784028    3.972664  \n",
       "               1                   0.626751     0.710366    5.995009  \n",
       "12             1                   0.503629     0.579437    5.775745  \n",
       "               3                   0.502110     0.587355    8.040761  \n",
       "48             3                   0.448354     0.504960    8.769443  \n",
       "24             3                   0.419409     0.456298    8.399888  \n",
       "1              1                   0.315781     0.288022    5.756575  \n",
       "               3                   0.181603     0.000000    0.857066  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psg = past_scores.sort_values(by='test_score', ascending=False).groupby(by=['no_of_features', 'hidden_layers'])\n",
    "psg.first().sort_values(by='test_score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.164631Z",
     "start_time": "2017-07-24T00:04:21.146124Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>epoch</th>\n",
       "      <th>train_score</th>\n",
       "      <th>test_score</th>\n",
       "      <th>f1_score</th>\n",
       "      <th>test_score_20</th>\n",
       "      <th>f1_score_20</th>\n",
       "      <th>time_taken</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>no_of_features</th>\n",
       "      <th>hidden_layers</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <th>1</th>\n",
       "      <td>6.500000</td>\n",
       "      <td>0.897222</td>\n",
       "      <td>0.857213</td>\n",
       "      <td>0.867953</td>\n",
       "      <td>0.743565</td>\n",
       "      <td>0.831326</td>\n",
       "      <td>2.974197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <th>3</th>\n",
       "      <td>3.500000</td>\n",
       "      <td>0.832044</td>\n",
       "      <td>0.799148</td>\n",
       "      <td>0.811015</td>\n",
       "      <td>0.642405</td>\n",
       "      <td>0.756604</td>\n",
       "      <td>2.463423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <th>1</th>\n",
       "      <td>5.428571</td>\n",
       "      <td>0.744218</td>\n",
       "      <td>0.761888</td>\n",
       "      <td>0.781667</td>\n",
       "      <td>0.685413</td>\n",
       "      <td>0.773989</td>\n",
       "      <td>3.160169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <th>1</th>\n",
       "      <td>6.250000</td>\n",
       "      <td>0.780754</td>\n",
       "      <td>0.724966</td>\n",
       "      <td>0.696372</td>\n",
       "      <td>0.581814</td>\n",
       "      <td>0.664497</td>\n",
       "      <td>3.535491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">12</th>\n",
       "      <th>3</th>\n",
       "      <td>6.250000</td>\n",
       "      <td>0.825794</td>\n",
       "      <td>0.708686</td>\n",
       "      <td>0.675391</td>\n",
       "      <td>0.477015</td>\n",
       "      <td>0.558293</td>\n",
       "      <td>4.720200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6.250000</td>\n",
       "      <td>0.829812</td>\n",
       "      <td>0.700591</td>\n",
       "      <td>0.662312</td>\n",
       "      <td>0.468038</td>\n",
       "      <td>0.535805</td>\n",
       "      <td>3.422357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <th>3</th>\n",
       "      <td>6.250000</td>\n",
       "      <td>0.823562</td>\n",
       "      <td>0.652286</td>\n",
       "      <td>0.551390</td>\n",
       "      <td>0.383924</td>\n",
       "      <td>0.396304</td>\n",
       "      <td>5.153420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <th>3</th>\n",
       "      <td>6.250000</td>\n",
       "      <td>0.723065</td>\n",
       "      <td>0.583293</td>\n",
       "      <td>0.477953</td>\n",
       "      <td>0.357458</td>\n",
       "      <td>0.356645</td>\n",
       "      <td>4.937698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"2\" valign=\"top\">1</th>\n",
       "      <th>1</th>\n",
       "      <td>6.250000</td>\n",
       "      <td>0.711806</td>\n",
       "      <td>0.513529</td>\n",
       "      <td>0.250800</td>\n",
       "      <td>0.272679</td>\n",
       "      <td>0.199582</td>\n",
       "      <td>3.410016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.551190</td>\n",
       "      <td>0.430758</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.181603</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.857066</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                 epoch  train_score  test_score  f1_score  \\\n",
       "no_of_features hidden_layers                                                \n",
       "48             1              6.500000     0.897222    0.857213  0.867953   \n",
       "122            3              3.500000     0.832044    0.799148  0.811015   \n",
       "24             1              5.428571     0.744218    0.761888  0.781667   \n",
       "122            1              6.250000     0.780754    0.724966  0.696372   \n",
       "12             3              6.250000     0.825794    0.708686  0.675391   \n",
       "               1              6.250000     0.829812    0.700591  0.662312   \n",
       "48             3              6.250000     0.823562    0.652286  0.551390   \n",
       "24             3              6.250000     0.723065    0.583293  0.477953   \n",
       "1              1              6.250000     0.711806    0.513529  0.250800   \n",
       "               3              2.000000     0.551190    0.430758  0.000000   \n",
       "\n",
       "                              test_score_20  f1_score_20  time_taken  \n",
       "no_of_features hidden_layers                                          \n",
       "48             1                   0.743565     0.831326    2.974197  \n",
       "122            3                   0.642405     0.756604    2.463423  \n",
       "24             1                   0.685413     0.773989    3.160169  \n",
       "122            1                   0.581814     0.664497    3.535491  \n",
       "12             3                   0.477015     0.558293    4.720200  \n",
       "               1                   0.468038     0.535805    3.422357  \n",
       "48             3                   0.383924     0.396304    5.153420  \n",
       "24             3                   0.357458     0.356645    4.937698  \n",
       "1              1                   0.272679     0.199582    3.410016  \n",
       "               3                   0.181603     0.000000    0.857066  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "psg.mean().sort_values(by='test_score', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.212685Z",
     "start_time": "2017-07-24T00:04:21.166134Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "Train.predictions = pd.read_pickle(\"dataset/tf_dense_only_nsl_kdd_predictions-.pkl\")\n",
    "Train.predictions_ = pd.read_pickle(\"dataset/tf_dense_only_nsl_kdd_predictions-__.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.243386Z",
     "start_time": "2017-07-24T00:04:21.214211Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual</th>\n",
       "      <th>Attack_prob</th>\n",
       "      <th>Normal_prob</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.315425</td>\n",
       "      <td>0.684575</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.313895</td>\n",
       "      <td>0.686105</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.518888</td>\n",
       "      <td>0.481112</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.492554</td>\n",
       "      <td>0.507446</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.407654</td>\n",
       "      <td>0.592345</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.766832</td>\n",
       "      <td>0.233168</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.624008</td>\n",
       "      <td>0.375992</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.676774</td>\n",
       "      <td>0.323226</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.746453</td>\n",
       "      <td>0.253547</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.533616</td>\n",
       "      <td>0.466384</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.519037</td>\n",
       "      <td>0.480963</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.661695</td>\n",
       "      <td>0.338305</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.315470</td>\n",
       "      <td>0.684530</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.415229</td>\n",
       "      <td>0.584771</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.556834</td>\n",
       "      <td>0.443166</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.770972</td>\n",
       "      <td>0.229028</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.736537</td>\n",
       "      <td>0.263463</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.770437</td>\n",
       "      <td>0.229563</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.583279</td>\n",
       "      <td>0.416721</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.319527</td>\n",
       "      <td>0.680473</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.386174</td>\n",
       "      <td>0.613827</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.598108</td>\n",
       "      <td>0.401892</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.738327</td>\n",
       "      <td>0.261673</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.772876</td>\n",
       "      <td>0.227124</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.277226</td>\n",
       "      <td>0.722774</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.535714</td>\n",
       "      <td>0.464286</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.741612</td>\n",
       "      <td>0.258388</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.740315</td>\n",
       "      <td>0.259685</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.569087</td>\n",
       "      <td>0.430913</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.578221</td>\n",
       "      <td>0.421779</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.536508</td>\n",
       "      <td>0.463492</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.681668</td>\n",
       "      <td>0.318332</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.775476</td>\n",
       "      <td>0.224524</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.470893</td>\n",
       "      <td>0.529107</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.280704</td>\n",
       "      <td>0.719296</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.247434</td>\n",
       "      <td>0.752566</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.739893</td>\n",
       "      <td>0.260107</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.537207</td>\n",
       "      <td>0.462793</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.573186</td>\n",
       "      <td>0.426814</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.492756</td>\n",
       "      <td>0.507244</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.334219</td>\n",
       "      <td>0.665781</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.777075</td>\n",
       "      <td>0.222925</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.763563</td>\n",
       "      <td>0.236437</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.726119</td>\n",
       "      <td>0.273881</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.592471</td>\n",
       "      <td>0.407529</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.730619</td>\n",
       "      <td>0.269381</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.517486</td>\n",
       "      <td>0.482514</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.741790</td>\n",
       "      <td>0.258210</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.319903</td>\n",
       "      <td>0.680097</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.545315</td>\n",
       "      <td>0.454685</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22494</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.500594</td>\n",
       "      <td>0.499406</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22495</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.782375</td>\n",
       "      <td>0.217624</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22496</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.421413</td>\n",
       "      <td>0.578587</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22497</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.315251</td>\n",
       "      <td>0.684749</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22498</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.316045</td>\n",
       "      <td>0.683955</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22499</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.722879</td>\n",
       "      <td>0.277121</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22500</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.319497</td>\n",
       "      <td>0.680503</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22501</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.791304</td>\n",
       "      <td>0.208696</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22502</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.546509</td>\n",
       "      <td>0.453491</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22503</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.359054</td>\n",
       "      <td>0.640946</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22504</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.275268</td>\n",
       "      <td>0.724732</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22505</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.468573</td>\n",
       "      <td>0.531427</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22506</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584382</td>\n",
       "      <td>0.415618</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22507</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.579053</td>\n",
       "      <td>0.420947</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22508</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.701545</td>\n",
       "      <td>0.298455</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22509</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.454361</td>\n",
       "      <td>0.545639</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22510</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.485994</td>\n",
       "      <td>0.514006</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22511</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.721500</td>\n",
       "      <td>0.278500</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22512</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.528028</td>\n",
       "      <td>0.471972</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22513</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.603016</td>\n",
       "      <td>0.396984</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22514</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.520138</td>\n",
       "      <td>0.479862</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22515</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.508554</td>\n",
       "      <td>0.491446</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22516</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.758068</td>\n",
       "      <td>0.241932</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22517</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.540403</td>\n",
       "      <td>0.459597</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22518</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.713313</td>\n",
       "      <td>0.286687</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22519</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.487434</td>\n",
       "      <td>0.512566</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22520</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.341756</td>\n",
       "      <td>0.658244</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22521</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.458776</td>\n",
       "      <td>0.541224</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22522</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.739913</td>\n",
       "      <td>0.260087</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22523</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.755810</td>\n",
       "      <td>0.244190</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22524</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.587982</td>\n",
       "      <td>0.412018</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22525</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.438880</td>\n",
       "      <td>0.561120</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22526</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.768690</td>\n",
       "      <td>0.231310</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22527</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.788431</td>\n",
       "      <td>0.211569</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22528</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.481061</td>\n",
       "      <td>0.518939</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22529</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.557205</td>\n",
       "      <td>0.442795</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22530</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.322497</td>\n",
       "      <td>0.677503</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22531</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.361536</td>\n",
       "      <td>0.638464</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22532</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.741776</td>\n",
       "      <td>0.258224</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22533</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.771269</td>\n",
       "      <td>0.228731</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22534</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.315719</td>\n",
       "      <td>0.684281</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22535</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.786737</td>\n",
       "      <td>0.213263</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22536</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.482062</td>\n",
       "      <td>0.517938</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22537</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.617349</td>\n",
       "      <td>0.382651</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22538</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.479014</td>\n",
       "      <td>0.520986</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22539</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.646830</td>\n",
       "      <td>0.353170</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22540</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.745547</td>\n",
       "      <td>0.254453</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22541</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.736529</td>\n",
       "      <td>0.263471</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22542</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.566808</td>\n",
       "      <td>0.433192</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22543</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.343301</td>\n",
       "      <td>0.656699</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>22544 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Actual  Attack_prob  Normal_prob  Prediction\n",
       "0         1.0     0.315425     0.684575         1.0\n",
       "1         1.0     0.313895     0.686105         1.0\n",
       "2         0.0     0.518888     0.481112         0.0\n",
       "3         1.0     0.492554     0.507446         1.0\n",
       "4         1.0     0.407654     0.592345         1.0\n",
       "5         0.0     0.766832     0.233168         0.0\n",
       "6         0.0     0.624008     0.375992         0.0\n",
       "7         1.0     0.676774     0.323226         0.0\n",
       "8         0.0     0.746453     0.253547         0.0\n",
       "9         1.0     0.533616     0.466384         0.0\n",
       "10        1.0     0.519037     0.480963         0.0\n",
       "11        0.0     0.661695     0.338305         0.0\n",
       "12        1.0     0.315470     0.684530         1.0\n",
       "13        1.0     0.415229     0.584771         1.0\n",
       "14        0.0     0.556834     0.443166         0.0\n",
       "15        0.0     0.770972     0.229028         0.0\n",
       "16        0.0     0.736537     0.263463         0.0\n",
       "17        0.0     0.770437     0.229563         0.0\n",
       "18        0.0     0.583279     0.416721         0.0\n",
       "19        1.0     0.319527     0.680473         1.0\n",
       "20        1.0     0.386174     0.613827         1.0\n",
       "21        1.0     0.598108     0.401892         0.0\n",
       "22        0.0     0.738327     0.261673         0.0\n",
       "23        0.0     0.772876     0.227124         0.0\n",
       "24        1.0     0.277226     0.722774         1.0\n",
       "25        1.0     0.535714     0.464286         0.0\n",
       "26        0.0     0.741612     0.258388         0.0\n",
       "27        0.0     0.740315     0.259685         0.0\n",
       "28        1.0     0.569087     0.430913         0.0\n",
       "29        0.0     0.578221     0.421779         0.0\n",
       "30        1.0     0.536508     0.463492         0.0\n",
       "31        0.0     0.681668     0.318332         0.0\n",
       "32        0.0     0.775476     0.224524         0.0\n",
       "33        0.0     0.470893     0.529107         1.0\n",
       "34        1.0     0.280704     0.719296         1.0\n",
       "35        1.0     0.247434     0.752566         1.0\n",
       "36        0.0     0.739893     0.260107         0.0\n",
       "37        1.0     0.537207     0.462793         0.0\n",
       "38        0.0     0.573186     0.426814         0.0\n",
       "39        0.0     0.492756     0.507244         1.0\n",
       "40        1.0     0.334219     0.665781         1.0\n",
       "41        0.0     0.777075     0.222925         0.0\n",
       "42        0.0     0.763563     0.236437         0.0\n",
       "43        0.0     0.726119     0.273881         0.0\n",
       "44        1.0     0.592471     0.407529         0.0\n",
       "45        0.0     0.730619     0.269381         0.0\n",
       "46        1.0     0.517486     0.482514         0.0\n",
       "47        1.0     0.741790     0.258210         0.0\n",
       "48        1.0     0.319903     0.680097         1.0\n",
       "49        0.0     0.545315     0.454685         0.0\n",
       "...       ...          ...          ...         ...\n",
       "22494     1.0     0.500594     0.499406         0.0\n",
       "22495     0.0     0.782375     0.217624         0.0\n",
       "22496     1.0     0.421413     0.578587         1.0\n",
       "22497     1.0     0.315251     0.684749         1.0\n",
       "22498     1.0     0.316045     0.683955         1.0\n",
       "22499     0.0     0.722879     0.277121         0.0\n",
       "22500     1.0     0.319497     0.680503         1.0\n",
       "22501     1.0     0.791304     0.208696         0.0\n",
       "22502     1.0     0.546509     0.453491         0.0\n",
       "22503     1.0     0.359054     0.640946         1.0\n",
       "22504     1.0     0.275268     0.724732         1.0\n",
       "22505     1.0     0.468573     0.531427         1.0\n",
       "22506     0.0     0.584382     0.415618         0.0\n",
       "22507     0.0     0.579053     0.420947         0.0\n",
       "22508     0.0     0.701545     0.298455         0.0\n",
       "22509     1.0     0.454361     0.545639         1.0\n",
       "22510     1.0     0.485994     0.514006         1.0\n",
       "22511     0.0     0.721500     0.278500         0.0\n",
       "22512     1.0     0.528028     0.471972         0.0\n",
       "22513     1.0     0.603016     0.396984         0.0\n",
       "22514     0.0     0.520138     0.479862         0.0\n",
       "22515     1.0     0.508554     0.491446         0.0\n",
       "22516     0.0     0.758068     0.241932         0.0\n",
       "22517     1.0     0.540403     0.459597         0.0\n",
       "22518     0.0     0.713313     0.286687         0.0\n",
       "22519     1.0     0.487434     0.512566         1.0\n",
       "22520     1.0     0.341756     0.658244         1.0\n",
       "22521     1.0     0.458776     0.541224         1.0\n",
       "22522     1.0     0.739913     0.260087         0.0\n",
       "22523     0.0     0.755810     0.244190         0.0\n",
       "22524     1.0     0.587982     0.412018         0.0\n",
       "22525     1.0     0.438880     0.561120         1.0\n",
       "22526     0.0     0.768690     0.231310         0.0\n",
       "22527     0.0     0.788431     0.211569         0.0\n",
       "22528     1.0     0.481061     0.518939         1.0\n",
       "22529     0.0     0.557205     0.442795         0.0\n",
       "22530     1.0     0.322497     0.677503         1.0\n",
       "22531     1.0     0.361536     0.638464         1.0\n",
       "22532     0.0     0.741776     0.258224         0.0\n",
       "22533     0.0     0.771269     0.228731         0.0\n",
       "22534     1.0     0.315719     0.684281         1.0\n",
       "22535     0.0     0.786737     0.213263         0.0\n",
       "22536     1.0     0.482062     0.517938         1.0\n",
       "22537     1.0     0.617349     0.382651         0.0\n",
       "22538     1.0     0.479014     0.520986         1.0\n",
       "22539     0.0     0.646830     0.353170         0.0\n",
       "22540     0.0     0.745547     0.254453         0.0\n",
       "22541     1.0     0.736529     0.263471         0.0\n",
       "22542     0.0     0.566808     0.433192         0.0\n",
       "22543     1.0     0.343301     0.656699         1.0\n",
       "\n",
       "[22544 rows x 4 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#epoch_nof_hidden\n",
    "Train.predictions[\"12_12_3\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.275611Z",
     "start_time": "2017-07-24T00:04:21.244742Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Actual</th>\n",
       "      <th>Attack_prob</th>\n",
       "      <th>Normal_prob</th>\n",
       "      <th>Prediction</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.543920</td>\n",
       "      <td>0.456081</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.526548</td>\n",
       "      <td>0.473452</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.496500</td>\n",
       "      <td>0.503500</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.579483</td>\n",
       "      <td>0.420517</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.319396</td>\n",
       "      <td>0.680604</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.705106</td>\n",
       "      <td>0.294894</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.197644</td>\n",
       "      <td>0.802356</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.515859</td>\n",
       "      <td>0.484141</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.575331</td>\n",
       "      <td>0.424669</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.472066</td>\n",
       "      <td>0.527934</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.358096</td>\n",
       "      <td>0.641904</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.362601</td>\n",
       "      <td>0.637399</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.482501</td>\n",
       "      <td>0.517499</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.542294</td>\n",
       "      <td>0.457706</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.499134</td>\n",
       "      <td>0.500866</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.206643</td>\n",
       "      <td>0.793357</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.493367</td>\n",
       "      <td>0.506633</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.346482</td>\n",
       "      <td>0.653518</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.507655</td>\n",
       "      <td>0.492345</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.448269</td>\n",
       "      <td>0.551731</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.506078</td>\n",
       "      <td>0.493922</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.539517</td>\n",
       "      <td>0.460483</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.445136</td>\n",
       "      <td>0.554864</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.804998</td>\n",
       "      <td>0.195002</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.505626</td>\n",
       "      <td>0.494374</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.541719</td>\n",
       "      <td>0.458281</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.741133</td>\n",
       "      <td>0.258867</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.596873</td>\n",
       "      <td>0.403127</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.579421</td>\n",
       "      <td>0.420579</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.050611</td>\n",
       "      <td>0.949389</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.474777</td>\n",
       "      <td>0.525223</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.799844</td>\n",
       "      <td>0.200156</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.578895</td>\n",
       "      <td>0.421105</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.521038</td>\n",
       "      <td>0.478962</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.547626</td>\n",
       "      <td>0.452374</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.461183</td>\n",
       "      <td>0.538817</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.436298</td>\n",
       "      <td>0.563702</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.513554</td>\n",
       "      <td>0.486447</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.396584</td>\n",
       "      <td>0.603416</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.227210</td>\n",
       "      <td>0.772790</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.657107</td>\n",
       "      <td>0.342893</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.723432</td>\n",
       "      <td>0.276568</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.553322</td>\n",
       "      <td>0.446678</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.591006</td>\n",
       "      <td>0.408994</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.296897</td>\n",
       "      <td>0.703103</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.637234</td>\n",
       "      <td>0.362765</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.355710</td>\n",
       "      <td>0.644290</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.542141</td>\n",
       "      <td>0.457859</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.148797</td>\n",
       "      <td>0.851203</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.364512</td>\n",
       "      <td>0.635488</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11800</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.616209</td>\n",
       "      <td>0.383791</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11801</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.285195</td>\n",
       "      <td>0.714805</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11802</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.567614</td>\n",
       "      <td>0.432386</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11803</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.806096</td>\n",
       "      <td>0.193904</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11804</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.793400</td>\n",
       "      <td>0.206600</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11805</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.497081</td>\n",
       "      <td>0.502919</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11806</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.582638</td>\n",
       "      <td>0.417362</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11807</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.818284</td>\n",
       "      <td>0.181716</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11808</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.228338</td>\n",
       "      <td>0.771662</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11809</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.402391</td>\n",
       "      <td>0.597609</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11810</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.694455</td>\n",
       "      <td>0.305545</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11811</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.462588</td>\n",
       "      <td>0.537412</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11812</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.563739</td>\n",
       "      <td>0.436261</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11813</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.680450</td>\n",
       "      <td>0.319550</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11814</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.852943</td>\n",
       "      <td>0.147057</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11815</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.513456</td>\n",
       "      <td>0.486544</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11816</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.803023</td>\n",
       "      <td>0.196977</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11817</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.582011</td>\n",
       "      <td>0.417989</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11818</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.613968</td>\n",
       "      <td>0.386032</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11819</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.743872</td>\n",
       "      <td>0.256128</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11820</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.542108</td>\n",
       "      <td>0.457892</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11821</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.613308</td>\n",
       "      <td>0.386691</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11822</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.598801</td>\n",
       "      <td>0.401199</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11823</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.366480</td>\n",
       "      <td>0.633520</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11824</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.569719</td>\n",
       "      <td>0.430281</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11825</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.718438</td>\n",
       "      <td>0.281562</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11826</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.638256</td>\n",
       "      <td>0.361744</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11827</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.306060</td>\n",
       "      <td>0.693940</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11828</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.785302</td>\n",
       "      <td>0.214698</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11829</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.583305</td>\n",
       "      <td>0.416695</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11830</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.575192</td>\n",
       "      <td>0.424808</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11831</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.492085</td>\n",
       "      <td>0.507915</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11832</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.149318</td>\n",
       "      <td>0.850682</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11833</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.495344</td>\n",
       "      <td>0.504656</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11834</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.491428</td>\n",
       "      <td>0.508572</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11835</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.582543</td>\n",
       "      <td>0.417457</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11836</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.597265</td>\n",
       "      <td>0.402735</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11837</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.621068</td>\n",
       "      <td>0.378932</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11838</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.386631</td>\n",
       "      <td>0.613369</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11839</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.404297</td>\n",
       "      <td>0.595703</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11840</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.499817</td>\n",
       "      <td>0.500183</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11841</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.590678</td>\n",
       "      <td>0.409322</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11842</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.547577</td>\n",
       "      <td>0.452423</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11843</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.359032</td>\n",
       "      <td>0.640968</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11844</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.803895</td>\n",
       "      <td>0.196105</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11845</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.584702</td>\n",
       "      <td>0.415298</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11846</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.720294</td>\n",
       "      <td>0.279706</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11847</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.408420</td>\n",
       "      <td>0.591580</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11848</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.342619</td>\n",
       "      <td>0.657381</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11849</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.542108</td>\n",
       "      <td>0.457892</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11850 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Actual  Attack_prob  Normal_prob  Prediction\n",
       "0         1.0     0.543920     0.456081         0.0\n",
       "1         1.0     0.526548     0.473452         0.0\n",
       "2         1.0     0.496500     0.503500         1.0\n",
       "3         0.0     0.579483     0.420517         0.0\n",
       "4         1.0     0.319396     0.680604         1.0\n",
       "5         1.0     0.705106     0.294894         0.0\n",
       "6         1.0     0.197644     0.802356         1.0\n",
       "7         1.0     0.515859     0.484141         0.0\n",
       "8         0.0     0.575331     0.424669         0.0\n",
       "9         0.0     0.472066     0.527934         1.0\n",
       "10        1.0     0.358096     0.641904         1.0\n",
       "11        1.0     0.362601     0.637399         1.0\n",
       "12        1.0     0.482501     0.517499         1.0\n",
       "13        1.0     0.542294     0.457706         0.0\n",
       "14        1.0     0.499134     0.500866         1.0\n",
       "15        1.0     0.206643     0.793357         1.0\n",
       "16        1.0     0.493367     0.506633         1.0\n",
       "17        1.0     0.346482     0.653518         1.0\n",
       "18        1.0     0.507655     0.492345         0.0\n",
       "19        1.0     0.448269     0.551731         1.0\n",
       "20        1.0     0.506078     0.493922         0.0\n",
       "21        1.0     0.539517     0.460483         0.0\n",
       "22        1.0     0.445136     0.554864         1.0\n",
       "23        1.0     0.804998     0.195002         0.0\n",
       "24        1.0     0.505626     0.494374         0.0\n",
       "25        1.0     0.541719     0.458281         0.0\n",
       "26        1.0     0.741133     0.258867         0.0\n",
       "27        1.0     0.596873     0.403127         0.0\n",
       "28        0.0     0.579421     0.420579         0.0\n",
       "29        1.0     0.050611     0.949389         1.0\n",
       "30        1.0     0.474777     0.525223         1.0\n",
       "31        1.0     0.799844     0.200156         0.0\n",
       "32        0.0     0.578895     0.421105         0.0\n",
       "33        0.0     0.521038     0.478962         0.0\n",
       "34        1.0     0.547626     0.452374         0.0\n",
       "35        0.0     0.461183     0.538817         1.0\n",
       "36        1.0     0.436298     0.563702         1.0\n",
       "37        1.0     0.513554     0.486447         0.0\n",
       "38        1.0     0.396584     0.603416         1.0\n",
       "39        1.0     0.227210     0.772790         1.0\n",
       "40        0.0     0.657107     0.342893         0.0\n",
       "41        1.0     0.723432     0.276568         0.0\n",
       "42        1.0     0.553322     0.446678         0.0\n",
       "43        0.0     0.591006     0.408994         0.0\n",
       "44        1.0     0.296897     0.703103         1.0\n",
       "45        1.0     0.637234     0.362765         0.0\n",
       "46        1.0     0.355710     0.644290         1.0\n",
       "47        1.0     0.542141     0.457859         0.0\n",
       "48        1.0     0.148797     0.851203         1.0\n",
       "49        1.0     0.364512     0.635488         1.0\n",
       "...       ...          ...          ...         ...\n",
       "11800     1.0     0.616209     0.383791         0.0\n",
       "11801     1.0     0.285195     0.714805         1.0\n",
       "11802     0.0     0.567614     0.432386         0.0\n",
       "11803     1.0     0.806096     0.193904         0.0\n",
       "11804     1.0     0.793400     0.206600         0.0\n",
       "11805     1.0     0.497081     0.502919         1.0\n",
       "11806     0.0     0.582638     0.417362         0.0\n",
       "11807     1.0     0.818284     0.181716         0.0\n",
       "11808     1.0     0.228338     0.771662         1.0\n",
       "11809     1.0     0.402391     0.597609         1.0\n",
       "11810     1.0     0.694455     0.305545         0.0\n",
       "11811     1.0     0.462588     0.537412         1.0\n",
       "11812     0.0     0.563739     0.436261         0.0\n",
       "11813     1.0     0.680450     0.319550         0.0\n",
       "11814     1.0     0.852943     0.147057         0.0\n",
       "11815     1.0     0.513456     0.486544         0.0\n",
       "11816     1.0     0.803023     0.196977         0.0\n",
       "11817     0.0     0.582011     0.417989         0.0\n",
       "11818     0.0     0.613968     0.386032         0.0\n",
       "11819     1.0     0.743872     0.256128         0.0\n",
       "11820     1.0     0.542108     0.457892         0.0\n",
       "11821     1.0     0.613308     0.386691         0.0\n",
       "11822     1.0     0.598801     0.401199         0.0\n",
       "11823     1.0     0.366480     0.633520         1.0\n",
       "11824     1.0     0.569719     0.430281         0.0\n",
       "11825     0.0     0.718438     0.281562         0.0\n",
       "11826     1.0     0.638256     0.361744         0.0\n",
       "11827     1.0     0.306060     0.693940         1.0\n",
       "11828     1.0     0.785302     0.214698         0.0\n",
       "11829     1.0     0.583305     0.416695         0.0\n",
       "11830     1.0     0.575192     0.424808         0.0\n",
       "11831     1.0     0.492085     0.507915         1.0\n",
       "11832     1.0     0.149318     0.850682         1.0\n",
       "11833     0.0     0.495344     0.504656         1.0\n",
       "11834     1.0     0.491428     0.508572         1.0\n",
       "11835     0.0     0.582543     0.417457         0.0\n",
       "11836     1.0     0.597265     0.402735         0.0\n",
       "11837     1.0     0.621068     0.378932         0.0\n",
       "11838     1.0     0.386631     0.613369         1.0\n",
       "11839     1.0     0.404297     0.595703         1.0\n",
       "11840     0.0     0.499817     0.500183         1.0\n",
       "11841     0.0     0.590678     0.409322         0.0\n",
       "11842     1.0     0.547577     0.452423         0.0\n",
       "11843     1.0     0.359032     0.640968         1.0\n",
       "11844     1.0     0.803895     0.196105         0.0\n",
       "11845     0.0     0.584702     0.415298         0.0\n",
       "11846     0.0     0.720294     0.279706         0.0\n",
       "11847     1.0     0.408420     0.591580         1.0\n",
       "11848     1.0     0.342619     0.657381         1.0\n",
       "11849     1.0     0.542108     0.457892         0.0\n",
       "\n",
       "[11850 rows x 4 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Train.predictions_[\"12_12_3\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.283529Z",
     "start_time": "2017-07-24T00:04:21.276895Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = Train.predictions[\"12_12_3\"].dropna()\n",
    "df_ = Train.predictions_[\"12_12_3\"].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.291114Z",
     "start_time": "2017-07-24T00:04:21.284852Z"
    },
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import metrics as me\n",
    "def get_score(y_true, y_pred):\n",
    "    f1 = me.f1_score(y_true, y_pred)\n",
    "    pre = me.precision_score(y_true, y_pred)\n",
    "    rec = me.recall_score(y_true, y_pred)\n",
    "    acc = me.accuracy_score(y_true, y_pred)\n",
    "    return {\"F1 Score\":f1, \"Precision\":pre, \"Recall\":rec, \"Accuracy\":acc}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.339064Z",
     "start_time": "2017-07-24T00:04:21.292448Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>F1 Score</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>Scenario</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.729595</td>\n",
       "      <td>0.704365</td>\n",
       "      <td>0.932580</td>\n",
       "      <td>0.565885</td>\n",
       "      <td>Train+/Test+</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.502110</td>\n",
       "      <td>0.587355</td>\n",
       "      <td>0.912826</td>\n",
       "      <td>0.432976</td>\n",
       "      <td>Train+/Test-</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Accuracy  F1 Score  Precision    Recall      Scenario\n",
       "0  0.729595  0.704365   0.932580  0.565885  Train+/Test+\n",
       "1  0.502110  0.587355   0.912826  0.432976  Train+/Test-"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import metrics as me\n",
    "\n",
    "scores = get_score(df.loc[:,'Actual'].values.astype(int),\n",
    "                df.loc[:,'Prediction'].values.astype(int))\n",
    "scores.update({\"Scenario\":\"Train+/Test+\"})\n",
    "score_df = pd.DataFrame(scores, index=[0])\n",
    "\n",
    "scores = get_score(df_.loc[:,'Actual'].values.astype(int),\n",
    "                df_.loc[:,'Prediction'].values.astype(int))\n",
    "scores.update({\"Scenario\":\"Train+/Test-\"})\n",
    "\n",
    "score_df = score_df.append(pd.DataFrame(scores, index=[1]))\n",
    "\n",
    "score_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.345847Z",
     "start_time": "2017-07-24T00:04:21.340444Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Actual\n",
       "0.0     9711\n",
       "1.0    12833\n",
       "Name: Actual, dtype: int64"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(by=\"Actual\").Actual.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.671428Z",
     "start_time": "2017-07-24T00:04:21.347243Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbQAAAGhCAYAAAAJL0FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd8FVX6x/HPk0IIvRcp0kWK0kFFVFTEBtgQy4r9tz/r\n6rqK5aeuu9jXVda2qKuwa2OxgAUVsYt0UKR3BGmhlxBSnt8fd4iXQEKAm4TM/b73dV+ZOXNm5lxw\nefKcOXOOuTsiIiKlXUJJN0BERCQWFNBERCQUFNBERCQUFNBERCQUFNBERCQUFNBERCQUFNBERCQU\nFNBERCQUFNBERCQUkkq6ASIiUnQSKx3pnpUek2t5+rpP3b13TC5WBBTQRERCzLPSSTmqf0yutXPG\nczVicqEiooAmIhJqBhYfT5fi41uKiEjoKUMTEQkzA8xKuhXFQhmaiEjYWUJsPoW5ldmtZvazmc0y\nsz8EZdXMbKyZLQh+Vo2qf7eZLTSzeWZ2RlR5RzObGRwbYrb/qKyAJiIiMWFmbYDrgC7AscA5ZtYM\nGASMc/fmwLhgHzNrBQwAWgO9gefNLDG43AvBtZoHn/2OrlRAExEJO7PYfPbvaGCiu+9w9yzga+B8\noC8wLKgzDOgXbPcF3nL3DHdfAiwEuphZXaCSu0/wyCrUw6POyZcCmohIqFksuxxrmNmUqM/1eW72\nM3CimVU3s3LAWUADoLa7rwrqrAZqB9v1gF+izl8RlNULtvOWF0iDQkREpLDS3L1TfgfdfY6ZPQZ8\nBmwHZgDZeeq4mXlRNE4ZmohI2BVflyPu/oq7d3T3HsBGYD6wJuhGJPi5Nqi+kkgGt1v9oGxlsJ23\nvEAKaCIiYWYU9yjHWsHPhkSen70BjAYGBlUGAqOC7dHAADNLMbPGRAZ/TAq6J7eYWbdgdOMVUefk\nS12OIiISS++YWXUgE7jR3TeZ2aPACDO7BlgG9Adw91lmNgKYDWQF9Xd3Ud4AvAakAmOCT4EsMoBE\nRETCKKFCXU9pO3D/FQth54THphb0DK2kKUMTEQk7zeUoIiJSeihDExEJuziZy1EBTUQk1LR8jIiI\nSKmiDE1EJMy0fIyIiEjpogxNRCTs4uQZmgKaiEioaVCIiIhIqaIMTUQk7BLiY1CIApqISJjtnm0/\nDsTHtxQRkdBThiYiEnZx8h6aApqISKhplKOIiEipogxNRCTs1OUoIiKhoC5HERGR0kMZmohImJnF\nTZejMjQREQkFZWgiImGnZ2gipZ+ZpZrZB2a22cz+ewjXuczMPotl20qKmZ1oZvNKuh1SjHZ3Ox7q\n5zCngCaHBTO71MymmNk2M1tlZmPMrHsMLn0hUBuo7u4XHexF3P11d+8Vg/YUKTNzM2tWUB13/9bd\njyquNokUF3U5Sokzs9uBQcDvgU+BXcAZQB/gu0O8/JHAfHfPOsTrhIKZJenPIt5ophCRYmFmlYGH\ngBvd/V133+7ume7+obvfGdRJMbOnzezX4PO0maUEx042sxVm9kczWxtkd1cFx/4M3A9cHGR+15jZ\ng2b2n6j7NwqymqRg/0ozW2xmW81siZldFlX+XdR5x5vZ5KArc7KZHR917Csz+4uZfR9c5zMzq5HP\n99/d/juj2t/PzM4ys/lmtsHM7omq38XMfjCzTUHdZ82sTHDsm6Daj8H3vTjq+neZ2Wrg1d1lwTlN\ng3t0CPaPMLN1ZnbyIf3FyuFFXY4ixeI4oCzwXgF17gW6Ae2AY4EuwH1Rx+sAlYF6wDXAc2ZW1d0f\nAB4G3nb3Cu7+SkENMbPywBDgTHevCBwPzNhHvWrAR0Hd6sBTwEdmVj2q2qXAVUAtoAxwRwG3rkPk\nz6AekQD8EnA50BE4Efg/M2sc1M0GbgNqEPmzOxW4AcDdewR1jg2+79tR169GJFu9PvrG7r4IuAv4\nj5mVA14Fhrn7VwW0V+SwpIAmJa06kLafbrDLgIfcfa27rwP+DPwu6nhmcDzT3T8GtgEH+4woB2hj\nZqnuvsrdZ+2jztnAAnf/t7tnufubwFzg3Kg6r7r7fHdPB0YQCcb5yQQGu3sm8BaRYPWMu28N7j+b\nSCDH3ae6+4TgvkuBfwInFeI7PeDuGUF79uDuLwELgYlAXSK/QEhY7F4PLRafw9zh30IJu/VAjd1d\nfvk4AlgWtb8sKMu9Rp6AuAOocKANcfftwMVEnuWtMrOPzKxlIdqzu031ovZXH0B71rt7drC9O+Cs\niTqevvt8M2thZh+a2Woz20IkA91nd2aUde6+cz91XgLaAP9w94z91JVSxRTQRIrJD0AG0K+AOr8S\n6S7brWFQdjC2A+Wi9utEH3T3T939dCKZylwi/9Dvrz2727TyINt0IF4g0q7m7l4JuIfI7+AF8YIO\nmlkF4GngFeDBoEtVpNRRQJMS5e6biTw3ei4YDFHOzJLN7Ewzezyo9iZwn5nVDAZX3A/8J79r7scM\noIeZNQwGpNy9+4CZ1TazvsGztAwiXZc5+7jGx0CL4FWDJDO7GGgFfHiQbToQFYEtwLYge/zfPMfX\nAE0O8JrPAFPc/VoizwZfPORWyuFFg0JEioe7/w24nchAj3XAL8BNwPtBlb8CU4CfgJnAtKDsYO41\nFng7uNZU9gxCCUE7fgU2EHk2lTdg4O7rgXOAPxLpMr0TOMfd0w6mTQfoDiIDTrYSyR7fznP8QWBY\nMAqy//4uZmZ9gd789j1vBzrsHt0pIREnXY7mXmBvhIiIlGIJVY70lJPu2X/FQtg5+vdT3b1TTC5W\nBPRitYhI2JWC7sJYUEATEQkz00whIiIipYoyNBGRsFOXY3ywpFS3MhVLuhkSAu2PbljSTZCQmDZt\napq71yzpdhwMM7sNuJbI+48ziUwBV47IiNxGwFKgv7tvDOrfTWTKumzgFnf/NCjvCLwGpBJ5VeZW\n388oRgW0MhVJOWq/o5tF9uv7ic+WdBMkJFKTLe9MNIfEiilDM7N6wC1AK3dPN7MRwAAi72mOc/dH\nzWwQkdU17jKzVsHx1kRm4PnczFoEM+e8AFxHZEq2j4m8XjKmoPvrGZqISIgZkYAWi08hJQGpwXR2\n5Yi819kXGBYcH8ZvMwP1Bd4K5hldQmRO0S5mVheoFMxb6sBwCp5NCFBAExGRwqthkYV4d3/yrt6w\nEngSWA6sAja7+2dAbXdfFVRbTWTRXYjMf/pL1CVWBGX1gu285QWK+y5HEZFQM/Y/22fhpRX0YrWZ\nVSWSdTUGNgH/NbPLo+u4u5tZkczooYAmIhJqB9RdeKhOA5YEyzxhZu8SWVdwjZnVdfdVQXfi2qD+\nSqBB1Pn1g7KVwXbe8gKpy1FERGJlOdAtmGTciCxAOwcYDQwM6gwERgXbo4EBFlmVvjHQHJgUdE9u\nMbNuwXWuiDonX8rQRERCrrgyNHefaGYjiUwgngVMB4YSWc9vhJldQ2TtwP5B/VnBSMjZQf0bo9YG\nvIHfhu2PYT8jHEEBTUQk9IqxyxF3fwB4IE9xBpFsbV/1BwOD91E+hciis4WmLkcREQkFZWgiIiFX\nnBlaSVJAExEJs9gO2z+sqctRRERCQRmaiEiIWfG+h1aiFNBEREIuXgKauhxFRCQUlKGJiIScMjQR\nEZFSRBmaiEjIxUuGpoAmIhJmeg9NRESkdFGGJiIScupyFBGRUi+eXqxWl6OIiISCMjQRkZCLlwxN\nAU1EJOziI56py1FERMJBGZqISJiZuhxFRCQk4iWgqctRRERCQRmaiEjIxUuGpoAmIhJierFaRESk\nlFGGJiISdvGRoClDExGRcFCGJiISZnoPTUREwiJeApq6HEVEJBSUoYmIhFy8ZGgKaCIiYRcf8Uxd\njiIiEg7K0EREQk5djiIiUuqZxc/UVwpoIhITRzVrRMUKFUlMTCQpKYnvJ04B4O67/sTHH31AmeQy\nNG7alKEvv0qVKlVYtnQp7doeTYsWRwHQpWs3/vH8iyX5FaSUU0ATkZj55PMvqVGjxh5lp552On8Z\n/AhJSUnce/ddPPHYIwx+5DEAmjRtysSpM0qiqXElXjI0DQoRkSJ12um9SEqK/O7cpWs3Vq5YUcIt\nij+7ux0P9XO4U0ATkZgwM84+4zSO79KRV14aus86w1/7F2f0PjN3f+mSJXTt2I7Te57Ed999W1xN\nlSJiZkeZ2YyozxYz+4OZVTOzsWa2IPhZNeqcu81soZnNM7Mzoso7mtnM4NgQK0REVZejiMTEuK++\no169eqxdu5Zzep/OUS1b0v3EHrnHH3tkMIlJSQy49DIA6tSty/zFy6levTrTpk6l/4X9mPbjLCpV\nqlRSXyG8iim5cvd5QDsAM0sEVgLvAYOAce7+qJkNCvbvMrNWwACgNXAE8LmZtXD3bOAF4DpgIvAx\n0BsYU9D9laGJSEzUq1cPgFq1atGn33lMnjwp99i/h73Gxx99yGvDX8/tukpJSaF69eoAdOjYkSZN\nmrJg/vzib7gUlVOBRe6+DOgLDAvKhwH9gu2+wFvunuHuS4CFQBczqwtUcvcJ7u7A8Khz8qWAJiKH\nbPv27WzdujV3+/Oxn9G6dRsAPvv0E5762+OMfG805cqVyz1n3bp1ZGdnA7Bk8WIWLlxA4yZNir/x\ncSCGz9BqmNmUqM/1Bdx2APBmsF3b3VcF26uB2sF2PeCXqHNWBGX1gu285QVSl6OIHLK1a9Zw8YXn\nAZCVncXFAy6l1xm9Abjt1pvIyMjgnN6nA78Nz//u22/4y5/vJzkpmYSEBP7x3ItUq1atxL5DaMV2\n+Zg0d++031ualQH6AHfnPebubmYeqwZFU0ATkUPWuEkTJk37cZ/HZs1duM/y886/gPPOv6AomyUl\n50xgmruvCfbXmFldd18VdCeuDcpXAg2izqsflK0MtvOWF0hdjiIiIWaAWWw+B+ASfutuBBgNDAy2\nBwKjosoHmFmKmTUGmgOTgu7JLWbWLRjdeEXUOflShiYiEmrF+w6ZmZUHTgf+J6r4UWCEmV0DLAP6\nA7j7LDMbAcwGsoAbgxGOADcArwGpREY3FjjCERTQREQkhtx9O1A9T9l6IqMe91V/MDB4H+VTgDYH\ncm91OcahGy85mSn/vYepI+/lpktPzi0//7T2TB15L9unDqFDq4a55UlJCbz00O+YPOIepr9zH3dc\n3Sv3WHJSIs/edwk/vX8/M969j36ntivOryIH6KhmjejUri1dO7aja8d2/DB+fIH1a1SpcMj3vO7q\nK2nZvDFdO7bjuM4dmPDDDwd8jQ8/GM0Tjz8KwOhR7zNn9uzcYw89eD9fjPv8kNsZZiXQ5VgilKHF\nmVZN63LV+cdz4u+eYFdmNqOfu4GPv/2Zxb+kMWvRrwz440s8e98le5xzwWkdSCmTROf+D5NaNpnp\n79zHiDFTWL5qA3ddewbrNmzlmH4PYWZUq1wunzvL4WJf8y0WtYcffYLzL7iQz8d+xs03/A+Tp/90\nQOefc24fzjm3DwAfjHqfM88+h6NbtQLg/gcfinl7w6Y0TFsVC8rQ4kzLxnWY/PNS0ndmkp2dw7dT\nF9KvZySrmrdkDQuWrd3rHMcpV7YMiYkJpKaUYVdmNlu37wRgYN/jeOJfn0XqubN+0/bi+zISE9u2\nbePMXqdyXOcOdGrXlg9G7/3sfdWqVZx2Sg+6dmxHx3Ztcqep+nzsZ5zU/TiO69yBSwdcxLZt2wq8\nV/cTe7BoUWTU448zZtDjhG50bn8M/S88j40bNwLw3D+G0P6YVnRufwy/u2wAEHkx+w+33MQP48fz\n0YejuWfQn+jasR2LFy3iuquv5N13RvLZp59w6YCLcu/1zddfcX7fcw6qnVI6KaDFmVmLfuWE9s2o\nVrk8qWWT6d29NfXrVC3wnHc/n86OnbtYMnYw88c8xNPDx7Fxyw4qV0gF4IEbz2H8G3fx+uNXU6ta\nxeL4GnIIep92Cl07tuPE47sCULZsWd4e+R4/TJ7GJ59/yaA7/0hkcobfvP3WG5ze6wwmTp3BpKk/\ncuyx7UhLS+PRh//Kx59+zg+Tp9GhYyeGPP1Ugff+6MMPaN2mLQDXXnUFgx95jMnTf6JNm7YM/suf\nAXjyiUeZMHk6k6f/xD+e23M5meOOP56zz+nDw48+wcSpM2jStGnusZ6nnsbkSRPZvj3yS9XIEW9z\nUf8BB9XOUIlRd2NpSPLU5Rhn5i1Zw99eG8sHz9/Ijp27+HHeCrKzcwo8p3PrRmRn59Ck171UrViO\nz/91G19MnMvW7TupX6cqE35czF1/e5dbLu/JI7edxzX/N7yYvo0cjLxdju7O/ffdw/fffkNCQgK/\nrlzJmjVrqFOnTm6dTp068z/XXU1mZibn9unHse3a8e03XzN3zmx69jgBgF2Zu+ja9bh93vOeQX/i\nsYf/So2aNXlx6Cts3ryZTZs3cWKPkwC4/HcDuSzIrtq2PYYrr7iMPn36cW7f/c52lCspKYlevXrz\n0YcfcP4FFzJmzEcMfvTxA2pnGBmQkFAKolEMKKDFoWHv/8Cw9yMP5v9807msXLOpwPr9z+zEZ+Nn\nk5WVw7qN2/hhxmI6tmrIO2Onsz09g/fHRV6ofXfsNAb2i59/KMLirTdeJy1tHeMnTSU5OZmjmjUi\nY+fOPep0P7EHY7/4hk8+/ojrr7mSW/5wO1WqVqXnaacz/D9v7vvCUXY/Q9tt8+bN+dZ9b/RHfPft\nN3z04Qc89uhgpkyfWejvctHFA3jh+WepVq0aHTp2omLFirh7odsppZu6HONQzaqRkWsN6lSlb89j\neXvMlALrr1i9gZM7R1YVLle2DF2OacS8pZEJAD7+5md6dGoOwMldjmLu4lX5XkcOT5s3b6ZmzVok\nJyfz9VdfsnzZsr3qLFu2jNq1a3P1tddx5dXXMn36NLp07cYP479n0cLIM7Ht27cXenLhypUrU7VK\n1dxncW+8/m+69ziJnJwcVvzyCyedfAqDH3mMzZs37/W8q0LFimwL5o3M68QeJzFj+jT+9cpLXNQ/\n8vztUNoZFupylNB688lrqValPJlZ2fzh0RFs3pYOQJ9TjuGpuy6iRtUKvDvk9/w0byV9bnyOF9/+\nhqF/vpypI+/FDP49agI/L/gVgPueeZ9X/jqQJ+64gLSN2/ifB/9Tkl9NDsKASy/jgn7n0qldWzp0\n7MRRLVvuVefbr7/i7089QXJSMuUrVOCVV4dTs2ZNXnrlNa64/BJ2ZWQA8MBDf6V5ixaFuu9L/xrG\nzTf+nvQdO2jUpAlDX36V7Oxsrhp4OVs2b8ZxbrjpFqpUqbLHeRf1H8CN/3sdzz87hDfeHrnHscTE\nRM486xz+M/w1Xv5XZHL3Q21nGMTLKEfL+/A33iSUq+UpR/Uv6WZICGyc/GxJN0FCIjXZphZmEuBC\nXatuC296zXOxuBSzBveKWbuKgjI0EZEwKyXdhbGgZ2giIhIKCmiHuW+G38GEtwYx/+OHWP7FI0x4\naxAT3hpEw7qxXTeqSYMapE9/lusu6p5bNuTeAQw4q3NM71O1UjmuvfC3e9SvXYV/P3pVTO8hhXfi\n8V3p2rEdzZs0pEHdmrlTYi1burRI7vfg/ffxj2eeBuCqKy5n9Kj396pz1RWX506V1bVjO049+cQi\naUu8iMy2H7MFPg9r6nI8zPW44kkALj+3Kx1bNeS2x/67z3oJCUZOzqE9D12dtoWbL+vJv94dv993\n0w5W1cqRgPbyyO8AWLFmE78b9GqR3Ev279vxE4HITBxTp07h6SGHx3PAx5/8O30KeActKyuLpKSk\nfPcLe158KB3BKBaUoZVSiYkJrPrmcZ644wImvX03nds0YuEnf8mdvaNL20Z89OJNAJRPLcPQP1/O\nt/++gx/evIuzeux7Aus167fw/fSFXHp2l72ONW1Yk9HP3cj3r9/J2Ff+QLOGtXLLvxl+B5NH3MOD\nN57Lqm8eB6Bi+bKM+efNjH/jLia9fTdnnhi5519v6UuLI2sx4a1B/OWWPjRpUIMJbw0C4LvX76T5\nkbVy7znuX7dxTIt6hW6/xM4rLw1l0J135O4PffEF7r7rTyxauJAOx7bmd5cNoF3bo7nskv6kp0dG\nyU6ZPJnTe57E8V060vecM1mzZk1+lz8oD95/H9dceQWn9DiB666+kldfeZmLLujHGaedwrlnnUFO\nTg533nE7Hdu1oVO7trz7TmQE5BfjPqfXqSdzft9z6NS+bUzbJIcXBbRSrErFcnw3bSFdLn6EiT8t\nybfePdefydjxczjxd09y5vVDePT280kps+/fUp98dSy3DTx1r9/onrvvEm595G1OuOxx7h8ymr8P\niszq8NSdF/H08HF07v8wq9N+e1k2PWMX/W9/ieMvfYyzf/8PHr/jfADuGzKK+cvW0m3Ao/zfkNF7\n3OOdT6dyQa8OANSrVYWqlcvx0/yVB9R+iY2LLh7A6FHvkZWVBcDwYa8y8MqrAZgzezY33fwHZsyc\nQ9mUsrw89J9kZGRwx+238uaIdxg/aSoDLr2chx74v4O+/5133Jbb5XjNlVfkls+bN5cxn43j1eGR\n10N+nDGdt/77LmM+G8c7I//LvLlzmDT1Rz78ZCx33nEba9dG5iadNnUKT//jeWbMnHPQbSrN9B7a\nITIzB55y9z8G+3cAFdz9waK65z7a8BrwobuP3F/d0ihjVyajvtj3svfRTj3uaHqd0Jo/XnU6AGXL\nJNGgTjUWLt97IuJFy9fx07yVXHRGh9yyyhVS6dK2EW8+eW1uWVJi5Hehzm0b0e/mFwB4e8wUHrgx\nMhmsYfzllj4c364pOe7Ur12V6lXKF9jOd8ZOY+TTv+fRlz7hwjM68O7Y6QfcfomNSpUq0b17Dz79\nZAyNGzchMTGRlkcfzaKFC2nUuDFdu3UD4JLLLueVl4fS46STmTN7FmefcRoA2dnZ1Ktf/6Dvn1+X\n47l9+lK2bNnc/dNO60XVqpG5SMd//x39L76ExMRE6tSpw/EndGfa1CmUKVOGrt2Oo2HDhntdL17E\nS5djUf6amwGcb2aPuHvagZ5sZknunlUE7QqN9IzMPfazsnNy52xLKZOcW24G/W8fypIVhftreOzl\nT3jtkSuZ9NPS3PPXb9pOtwGPFrptl53bhcoVUjnu0sfIzs5h4Sd/oWxUm/Zl+aqNbE/PoGWTOlzY\nqwPXPfCfg2q/xMaVV1/LkGee4sgjG3HFwN8G7uT9x9HMcHfatD2GcV99W6RtKlduz1+KypUv+Jek\nA60npVtRdjlmAUOB2/IeMLNGZvaFmf1kZuPMrGFQ/pqZvWhmE4HHzexBMxtmZt+a2TIzO9/MHjez\nmWb2iZklB+fdb2aTzexnMxtq8fLrSB7Lft1A+6Mjv4Wed9pvC21+Pn4ONww4KXf/2KMK/s15zuLV\nLPkljTNOiKw3tWlrOqvTNtPnlGOAyD9gbVvUA2DKz8vo2/NYAC46o2PuNSpXSGXdhq1kZ+fQs2tL\n6tWO/Ba9bXsGFcul5HvvkZ9O409X9aJMmSTmLl59UO2X2Dj+hBNYsmgR777zXy7sf3Fu+dIlS5gy\neTIAb7/5Bscf352jW7Xi119XMnnSJAB27drF7FmzirW9J3Q/kf+OeIucnBzWrFnDD+O/p0PHw/Yd\n4OITR7PtF/UztOeAy8yscp7yfwDD3P0Y4HVgSNSx+sDx7n57sN8U6An0Af4DfOnubYF04OygzrPu\n3tnd2wCpwDlF8m0Oc3998WOeuac/3/3nT+zK/C25HfzPMZRLLcPkEZFVqu/9/Vn7vdajL39Cg6hX\nA3436FWuvfBEJr49iGkj780d5PHHx//LH688jUlv302jetXZsi0yqe0bH06i27FNmDziHi7q3SF3\nnbW1G7Yyfc4vTB5xD3+5pc9e93338+lcfGYn3vls+iG1X2LjvAsupHv3HlSu/Nv/hVsefTRDnnmK\ndm2PZkf6Dq657npSUlJ4462R3PWn2+nc/hi6dW7P5EkTD/q+0c/QunZsR3Z29n7POf+CC2lxVEs6\ndziGs884jceeeIpatWrt97ywi6dh+0U29ZWZbXP3Cmb2EJBJJABVcPcHzSwNqOvumUGWtcrdawTP\nvL5092HBNR4EMt19sJklBNco6+4eXHeDuz9tZhcAdwLlgGrAP9z90fyeoZnZ9cD1ACRX6Fi29cAi\n+TOIB+XKlmHHzl0ADDirM317Hssld7xcwq0qGWGc+qrP2b3501135y7zsmjhQi69+EImTp1Rwi0L\nt1hOfVW+3lHe8vcv7r9iIUy7v2fcT331NDANKOzLRnmXPM4AcPccM8v03yJwDpBkZmWB54FO7v5L\nEATLUgB3H0qkO5SEcrXiezLLQ9Sx9ZE88acLSDBj09YdXP+AJicOg/Xr13NS92506NgpN5hJ6VUK\nkquYKPKA5u4bzGwEcA3wr6B4PDAA+DdwGXAoT5J3B680M6sAXAiEclTj4ejbqQsOaLCIlA7Vq1fn\n5zkL9ipv2qyZsrNSqDR0F8ZCcb2H9jegRtT+zcBVZvYT8Dvg1oO9sLtvAl4CfgY+BSYfQjtFRKSU\nKrIMzd0rRG2vIfJ8a/f+MiIDPfKec2We/QcLuOaDUdv3Afft73oiIvEoThI0zeUoIhJqpi5HERGR\nUkUZmohIiEXeQyvpVhQPZWgiIhIKytBEREKtdMzyEQsKaCIiIRcn8UxdjiIiEg7K0EREQk5djiIi\nUvqVkqVfYkFdjiIiEgrK0EREQmz3emjxQAFNRCTk4iWgqctRRERCQQFNRCTkzGLzKdy9rIqZjTSz\nuWY2x8yOM7NqZjbWzBYEP6tG1b/bzBaa2TwzOyOqvKOZzQyODbFCpJkKaCIiIWdmMfkU0jPAJ+7e\nEjgWmAMMAsa5e3NgXLCPmbUisthza6A38LyZJQbXeQG4DmgefHrv78YKaCIiEhNmVhnoAbwC4O67\ngkWY+wLDgmrDgH7Bdl/gLXfPcPclwEKgi5nVBSq5+wR3d2B41Dn5UkATEQmzGHU3BglaDTObEvW5\nPs/dGgPrgFfNbLqZvWxm5YHa7r4qqLMaqB1s1wN+iTp/RVBWL9jOW14gjXIUEZHCSnP3TgUcTwI6\nADe7+0Qze4age3E3d3cz86JonDI0EZEQM2Lz/KyQz9BWACvcfWKwP5JIgFsTdCMS/FwbHF8JNIg6\nv35QtjIi1tbwAAAgAElEQVTYzlteIAU0EZGQK65Rju6+GvjFzI4Kik4FZgOjgYFB2UBgVLA9Ghhg\nZilm1pjI4I9JQffkFjPrFoxuvCLqnHypy1FERGLpZuB1MysDLAauIpI8jTCza4BlQH8Ad59lZiOI\nBL0s4EZ3zw6ucwPwGpAKjAk+BVJAExEJuYRinCnE3WcA+3rOdmo+9QcDg/dRPgVocyD3VkATEQm5\nOJn5Ss/QREQkHJShiYiEWGRAR3ykaApoIiIhlxAf8UxdjiIiEg7K0EREQk5djiIiEgpxEs/U5Sgi\nIuGgDE1EJMSMyHyO8UABTUQk5DTKUUREpBRRhiYiEmaFX/ql1FOGJiIioaAMTUQk5OIkQVNAExEJ\nM6N4l48pSepyFBGRUFCGJiIScnGSoCmgiYiEnUY5ioiIlCLK0EREQiyywGdJt6J4KKCJiIScRjmK\niIiUIsrQRERCLj7yswICmplVKuhEd98S++aIiEisxcsox4IytFmAs2dw373vQMMibJeIiMgByTeg\nuXuD4myIiIjEXmTqq5JuRfEo1KAQMxtgZvcE2/XNrGPRNktERGIiWD4mFp/D3X4Dmpk9C5wC/C4o\n2gG8WJSNEhEROVCFGeV4vLt3MLPpAO6+wczKFHG7REQkRkpBchUThelyzDSzBCIDQTCz6kBOkbZK\nRETkABUmQ3sOeAeoaWZ/BvoDfy7SVomISMyUhudfsbDfgObuw81sKnBaUHSRu/9ctM0SEZFYiKdR\njoWdKSQRyCTS7ajpskRE5LBTmFGO9wJvAkcA9YE3zOzuom6YiIjERrwM2y9MhnYF0N7ddwCY2WBg\nOvBIUTZMRERi4/APRbFRmO7DVewZ+JKCMhERkcNGQZMT/53IM7MNwCwz+zTY7wVMLp7miYjIoTAr\n3vXQzGwpsBXIBrLcvZOZVQPeBhoBS4H+7r4xqH83cE1Q/xZ3/zQo7wi8BqQCHwO3ursXdO+Cuhx3\nj2ScBXwUVT6h8F9NRERKWgk8/jrF3dOi9gcB49z9UTMbFOzfZWatgAFAayLjND43sxbung28AFwH\nTCQS0HoDYwq6aUGTE79yKN9GREQk0Bc4OdgeBnwF3BWUv+XuGcASM1sIdAmyvEruPgHAzIYD/TjY\ngLabmTUFBgOtgLK7y929xQF9HRERKRHFPELRiWRa2cA/3X0oUNvdd4+9WA3UDrbrsWev34qgLDPY\nzlteoMKMcnwN+CvwJHAmcFXQYBERKQViGM9qmNmUqP2hQcCK1t3dV5pZLWCsmc2NPujubmZFEkMK\nM8qx3O6HdO6+yN3vIxLYREQkvqS5e6eoT95ghruvDH6uBd4DugBrzKwuQPBzbVB9JRC99mb9oGxl\nsJ23vECFCWgZweTEi8zs92Z2LlCxEOeJiEgJM4wEi81nv/cyK29mFXdvExkV/zMwGhgYVBsIjAq2\nRwMDzCzFzBoDzYFJQffkFjPrZpH+0iuizslXYbocbwPKA7cQeZZWGbi6EOeJiEh8qQ28FzyzSwLe\ncPdPzGwyMMLMrgGWEZnkHnefZWYjgNlAFnBjMMIR4AZ+G7Y/hv0MCNl9wwK5+8Rgcyu/LfIpIiKl\ngRXfsH13Xwwcu4/y9cCp+ZwzmEiylLd8CtDmQO5f0IvV71HA4A93P/9AbiQiIiWjNMzDGAsFZWjP\nFlsrSlClWjU4+cYrS7oZEgJNb3mvpJsgEtcKerF6XHE2REREika8rPlV2PXQRESkFDLip8sxXgK3\niIiEXKEzNDNLCebbEhGRUiQhPhK0Qq1Y3cXMZgILgv1jzewfRd4yERGJiQSLzedwV5guxyHAOcB6\nAHf/ETilKBslIiJyoArT5Zjg7svyPFTMzq+yiIgcPsziZ1BIYQLaL2bWBXAzSwRuBuYXbbNERCRW\nSkN3YSwUpsvxf4HbgYbAGqBbUCYiInLYKMxcjmuJLJEtIiKlUJz0OBZqxeqX2Mecju5+fZG0SERE\nYsagUEu/hEFhnqF9HrVdFjgP+KVomiMiInJwCtPl+Hb0vpn9G/iuyFokIiIxFS9TQh3M92xMZBE3\nERGRw0ZhnqFt5LdnaAnABmBQUTZKRERiJ04eoRUc0CzyNt6xwMqgKMfd8130U0REDi9mFjeDQgrs\ncgyC18funh18FMxEROSwVJhnaDPMrH2Rt0RERIpEZPqrQ/8c7vLtcjSzJHfPAtoDk81sEbCdyGsN\n7u4diqmNIiJyCOJl6quCnqFNAjoAfYqpLSIiIgetoIBmAO6+qJjaIiIiMaaZQiJqmtnt+R1096eK\noD0iIhJjcRLPCgxoiUAFgkxNRETkcFZQQFvl7g8VW0tERCT2TINCQJmZiEgoWJz8c17Qe2inFlsr\nREREDlG+GZq7byjOhoiISOxFRjmWdCuKR2HWQxMRkVIsXgJavCyTIyIiIacMTUQk5CxOXkRThiYi\nIqGgDE1EJMQ0KERERMKhlCz9EgvqchQRkVBQhiYiEnLxMtu+MjQRkRDb/QwtFp9C39Ms0cymm9mH\nwX41MxtrZguCn1Wj6t5tZgvNbJ6ZnRFV3tHMZgbHhlghhmoqoImISKzdCsyJ2h8EjHP35sC4YB8z\nawUMAFoDvYHnzSwxOOcF4DqgefDpvb+bKqCJiIScWWw+hbuX1QfOBl6OKu4LDAu2hwH9osrfcvcM\nd18CLAS6mFldoJK7T3B3B4ZHnZMvPUMTEQk1IyF2s+3XMLMpUftD3X1onjpPA3cCFaPKarv7qmB7\nNVA72K4HTIiqtyIoywy285YXSAFNREQKK83dO+V30MzOAda6+1QzO3lfddzdzcyLonEKaCIiIWYU\n63toJwB9zOwsoCxQycz+A6wxs7ruviroTlwb1F8JNIg6v35QtjLYzlteID1DExEJsxiNcCzMKEd3\nv9vd67t7IyKDPb5w98uB0cDAoNpAYFSwPRoYYGYpZtaYyOCPSUH35BYz6xaMbrwi6px8KUOLQ0Mv\nbkt6ZjY5Dtk5zh2jIoORBnQ4gtOPqsGWnVkA/GfySqau2EyPptU475g6uecfWS2VP743myUb0rms\nUz1OaVad8imJXDJseol8HxE57D0KjDCza4BlQH8Ad59lZiOA2UAWcKO7Zwfn3AC8BqQCY4JPgRTQ\n4tR9H81na0bWXuWjf17DqJlr9ij7ZtEGvlkUWe/1yKqp3H16U5ZsSAdg8rJNfDxrLc/3b1P0jRaR\ng1ISL1a7+1fAV8H2euDUfOoNBgbvo3wKcED/sCigyQE5sWk1vl28MXd//rrtJdgaEZHfKKDFIQce\nOqsFOe58Omcdn81Lyz12dqtanNK8OgvX7eDVib+wfVf2Hud2b1KVh8cuLOYWi8jBKuZBISVKAS0O\n3f3BXDbsyKRy2SQePLMFKzbvZPbqbYyZs5YR03/FHS7tVI+rujbg2W+X5p7XvGZ5MrJyWL5xZ8k1\nXkQOmOZylNDasCMTgM07s5i4bBPNa5aP7KdnkeORDG7s3HW55bud2KQa3wbP0kREDjcKaHEmJSmB\nsskJudvt6lVi+cbIAI+qqcm59bo2qppbDpFuixOaVOXbxQpoIqVNcU59VZLU5RhnqqQmMei0ZgAk\nJhjfLNrA9BVbABjYpT6Nq6fiwNqtu3jhu2W557WuW5G07btYs3XXHtcb2KU+JzatRkpSAi9fcgyf\nz0vjrWm/Ftv3EZGCGfGTuSigxZk1W3dx23uz93ns6a+X5Hvez6u2ctfouXuVD5u0gmGTVuzjDBGR\n4qWAJiISZgaFWEosFBTQRERCLj7CWfx0rYqISMgpQztMRc+3CPDi98uYtzb/WTneHNj+kOdSvKVH\nI46tV4n/eXsmWTlOxZQk/tbvaK5/e+YhXTevrkdWYeXmnazYFHmf7ZIORzBr9VZ++nVrTO8jRadp\nrQq8cE3n3P2GNcrz5IdzePnLRdx3XhtOb1uHXdk5LFu3ndv/PY0t6ZFXRY6uV4nHLmlPhbJJ5Lhz\n9mNfYWYMva4LR9YoT3aOM3bmah4ZNaukvlroGPHzHpoC2mEsv/kWi1KOO6cdVYNP5qwrsnt0PbIK\nk5dvzg1ob2pUZKmzaO02ej3yJRCZhX3qw2cy5sfI3+M3c9fyyKhZZOc49/RrzU1ntODh92eRmGAM\nubITt742hdkrt1C1fBkys3Mok5TIi58vYPz8NJITjbdv7c4prWrz5ew1BTVBDkB8hDMFtFKlbFIC\n95zejPIpiSQlGK9P+ZVJyzftUadqajJ39GxCuTKJJJjxz++XMXvNNtrVq8SADkeQnGis3pLBP75Z\nys6snL3u8cHPazm3TW0+m7t3QOvXtjYnNKlGcqIxYemm3OH5/dvV5aRm1dm8M5O07btYlLaDUTPX\ncPpRNejVsiZJCZF7/v2rJTSunkrnhlVoXbci/dvX5bHPF9G/fV0mL9/MzqxsTmtRgye+WAxAm7oV\n6du2NoM/W1jo9kvx696yFsvStrMymLD6mzlrc49NW7KBs9tHFho+6ehazFm5mdkrI6+JbNweeQVk\nZ2Y24+dHpl/LzHZm/rKJulVTi/MrSEgooB3G/np2C3IcMrNzuHP0XHZl5/DI5wtJz8yhYkoSj/dp\nuVdA69GsGtNXbmHkjFUkGJRJSqBiShIXtavLA2Pmk5GVw3nH1KFP29qMmL5qr3uu27aLOau3cXKz\n6kxevjm3vF29ShxRuSx/GjUHA+7p1YxWdSqwKyuH4xpX5Q/vRX4Df6pfKxal7QBgwtKNjA3miby0\nY2Rpmo9mr2Xy8k1MXr6ZH5Zu3OPeP67cwg3djyQlKYGMrBxOaFKV7xZvOKD2S/Hr27E+70/Z96sb\nA44/ktFTI+syNqlVARxev+l4qldIYdTUFbwwdsEe9SulJnN627q88sWiIm93PImTHkcFtMNZ3i5H\nw7i8U31a162AO1QrX4YqqUlsSv+tzoJ127n5xEYkJRgTl25kyYZ02jSoSIOqZXn03JYAJCUY89Zu\ny/e+7/y4intOb8bUX/YMaO3qVeLv57UCItniEZXKkpqcwMRlm8jMdjKznclRAbZh1VQu61SP8mUS\nKZucyPQVm/e6V7Qch+krttC5YWXGL9lIpwaVGTZpBW3qHFj7pfgkJxq9jqmzz2det/RuQVa28+6k\nXwBITDQ6N63OWY99RfqubEbc2p2Zyzfx3bxIb0BigvHc1Z3415eLWL5+R7F+j3AzDduXw89JzapR\nOTWJP743h2x3hl7cljKJew5Unb16G/d8NI9ODSpzy0mNGTVzDdt3ZTFj5Rae+jL/F6ejrdqSwZL1\nOzihSdXcMjMY+eMqPpubtkfdc1vXyvc6t5zUmEfGLmTphnR6Nq9Om7oV93vvbxdt4KzWtdiWkc3C\ntB3szMzBjANqvxSfU1rXYeYvm0jbmrFHef9uDTmtTV36P/NdbtmqjelMXLg+t6vxi1mradOgSm5A\ne/zS9ixZu52Xv1R2JgdHw/ZLkXJlEtmcnkm2O23qVqRWxZS96tSsUIbN6ZmMnZfG2HnraFqjHPPW\nbufo2hWoUylSPyUpgSMq7X1utP/OWEW/tr+tUj19xRZOa1GDskmR/2SqlUumctkk5qzZRueGlUlO\nNMomJdCpYZXcc1KTE9i4I5NEM3o0q55bnp6ZTWryvv/Tm7V6K02rl+P0o2rwXTAR8sG0X4pHv071\neX/ynt2NJ7eqxf+e3pwrX/yBnZm/LT/09ey1tDyiEmWTE0lMMLo1r8GC1ZHnaXeeezQVU5N4YORP\nxdr+eLB76qtYfA53ytBKka8XbuDeXs145vxWLEzbwS+b0veq06ZuRfq1rUN2jrMzK5unv1rClp1Z\nDPl6KX88pQnJiZGuh9enrOTXLRl7nb/bL5t2smj9DppWLwdEMqT6VcryWJ9It196Zg5Pf7WEhWk7\nmLx8M0+f35rN6Zks35DOjmANtTem/srjfY9mc3oWC9ZtIzU5EYBvF2/kxu5Hck7r2jw+bs/fxnMc\nJi/fTM8W1Xnm66UAB9V+KXqpZRLp0bIWd72x5+sif+1/LCnJCbx18wkATFu6kUFvzmBzeiZDv1jI\nx3edjBPJ0Mb9vIa6Vcpy65ktWbB6K58OOgWAV79ezJvjl+W9pUiBzN1Lug0lqkqjVn7yfcNLuhml\nWtmkBHZm5VAmMYGHzzmK579bxuI4fAYyedovJd0ECYlfXzh/qrt3isW1mrY61h95Y0wsLsXF7evF\nrF1FQRmaHLIbuh9Jg6qpJCcaXy5YH5fBTORwFh9DQhTQJAae+kqDNUSk5CmglVKP92lJcmICFVIS\nKZOYkLsK9SNjF7J22679nH3gLu14BFt3ZvHBrLV7lZ/aogZbdv726sDdH85lZ6Zeej6cffCnk0hJ\nSqBK+TKUTU5kdfA89up/TmTFhthl2I1qlufze09l8ZqtJCclMH5+Gve+/eMBX+f1m47n+pcmkZxo\nnNuxHv/+dikAR1RN5f/Ob8P/vjI5Zm0OHc22L4e7O4O1yXo2r07TGuV56YflJdaW939avVegi5Zg\n5M5Jua/9ghgQ3095i8a5T3wNRIbXH9OwCveN2PfowgP5u8rP4jVb6fXIlyQlGCNvO5HT29Zh7MzV\nB3SNy54dD0QC5O+6N84NaL9uTFcw2w8t8CmlVq+janBE5bK8Fiy62fvomtSumMKnc9dxz+nNWLYh\nnUbVU1m+IZ1nvl7CrmynWY1yXNm1AanJCWxKz2LIN0vYnH5oc0iedlQNOjeoTLkyibhH3mHr3/4I\n0jOzqVMphZtHzuK8Y+pwcjCc/7O56/ho9lrqVErh3tObsXj9DppUL8cDY+bnZp9S9BITjJmPn8V/\nJyzn+KNqMuiNGfzz2i70/Os4tqRn0qFRVe7s04oBQ76nXEoig/sfS/O6FUlOTODJD+cUGKiycpyp\nSzbQuGYFzOD+89vS4+hauDt//3guH03/lTqVy/LCNV0on5JIYmICd70xnSmLNzBlcG96/nUc9/Rt\nTZPaFfns7lP4cvZa3hy/lKHXdqHXI1/y8V0nc/OrU1gUvHT/3u09uG/EjyxZt+2A2imllwJayHy7\neAN/P68VwyevIMehZ/MaDPkm8oyrYdVUnv1mKfPXbefWkxpxRsuajJmzjmuPa8jgzxayNSOLHk2r\ncVnHejz/XeGHTPc7pg49W9QAIkPsHxgzH4DG1ctx23uz2b4rm2OOqEjTGuW4eeQs0rbvonnN8vRo\nWo0/jZpNQoLxRN+j+XnVVjKyc6hXpSxPf70kdwotKV6Vy5VhwsL1PDCy4FUWbjurJV/OXsNt/55G\n5dRkPrzzZL6Zs5aMfObYTC2TyAktajL4/Z85t0M9mtWpwOmDx1G9Ygof33UyExakcX6XBoyduYrn\nxy4gwaBsmcQ9rvHwqFk0qlk+d2LkRjXL5x4bPXUl53asx9Nj5lG3SlmqlE9m1orN3Hte6wNqZxip\ny7GImFk/4D3gaHefa2aNgOPd/Y3geDvgCHf/+CCvvxTo5O5p+6sbRumZOcxavY2ODSqzeksGOe6s\n2LSTOpVSWL0lg/nrIkvQfL1wA71a1uDnVVtpULUsD53VAoh0MaVtP7CMKL8uxxkrt7B9128v1s5b\nu520YJaIVnUq8MPSjezKdsh2Ji7dRKs6FZi+cgurt2QomJWgjMxsxszY/woIJ7WsxSmtanNTr8h/\nOynJCdSrVo7FeaYl251R5Th88uOvfDt3HX/pfwyjpkR+6Vq3JYNJC9dz7JFV+XHZRh67tD0pyYl8\n+uOvuRMZF8YH01bw2v8ex9Nj5tGnY30+nLbygNoZZvERzkomQ7sE+C74+QDQCLgUeCM43g7oBBxU\nQBMYOy+Nvm1qs3ZbBl/Mj47rez4McQcMlm1I554P58W8HXl/A86ImjXiQM6T4rUzz99TVk4OCcG/\niCnJv2VMZsY1/5zIsrT81+mD356hFcb389O48O/fcmqbOjwzsBPPj53Pe5P3PfFxXis3pLMjI4vm\ndSpybsd63DZ82gG1U0q/Yn1WaGYVgO7ANcCAoPhR4EQzm2FmdwEPARcH+xebWRcz+8HMppvZeDM7\nKrhWopk9aWY/m9lPZnZznnulmtkYM7uuGL/iYWHumm3UqZTC8Y2r8t3i32a0r1UxhWY1IjN/9Gha\njTlrtvHLxp1UK1eG5kHXTVKC0aBK2SJv4+zV2+h2ZFXKBFNmdT2yCrNXx89vzKXJL+t30DaY0uys\n9kfkln81Zw1Xn9wkd791/cqFvuakhevp07E+ZlCjYgqdm1bnx2UbqVctlbVbdvL690t5+4dltGlQ\nZY/ztu3MonzZ/H8PHz11JTef0YKUpEQWrN56yO0MC7PYfA53xZ2h9QU+cff5ZrbezDoCg4A73P0c\nADNbQ6TL8KZgvxJwortnmdlpwMPABcD1RLK7dsGxalH3qQC8BQx397icBmT8ko3Uq1KWHVG/ba/Y\ntJM+bWvTuHo5lm9I57O568jKcR4ft4hrj2uQu4baqJmr+SVYfLMwop+hAQz+bEEBtSMWrNvOt4s3\n8ETfyOz9n8xZx7KN6bnzNcrh46mP5vLEZe3ZnJ7JxAVpe5T/+cJj+PzeniSYsXTddq7+54RCXfPD\n6Svp0Lgan997Ku7On9+Zyfptu7j4uCO5/tRmZGXnsG1nFrcMm7LHeWlbM5i5fBOf39uTcT+v4c3x\nS/e87rSVPHBBW578cHZM2hkGkVGOpSAaxUCxTn1lZh8Cz7j7WDO7BWgIfMieAe1K9gxoDYAhQHMi\nfWbJ7t7SzN4BXnT3sXnusRTYDDzu7q/n047riQREUqvV6djrsQ9i/l1L2v1nNOedH1cxK8h66lRK\n4a5Tm3Lbe7P3c6YcLE19JbESy6mvmrc+1p9667NYXIo+x9TR1FcAQQbVE2hrZg4kEglQH+3n1L8A\nX7r7ecEAkq8Kcbvvgd5m9obvI2K7+1BgKETmcizsdygNKqYk8lifo1mYtj03mIlIfCsN3YWxUJzP\n0C4E/u3uR7p7I3dvACwBcoDohbK25tmvDKwMtq+MKh8L/I+ZJUFuwNztfmAj8FxMv0EpsDUjmxv+\n+/Nea4et3pKh7EwkLlnM/ne4K86AdgmR4frR3iEyOCTbzH40s9uAL4FWuweFAI8Dj5jZdPbMKF8G\nlgM/mdmPREZKRrsVSDWzx4vgu4iIyGGm2Loc3f2UfZQNyad65zz7LaK27wvOzQJuDz7R12wUtXvV\nATdURCRk1OUoIiJSimjqKxGREIunYfsKaCIiYVZKXoqOBXU5iohITJhZWTObFAzym2Vmfw7Kq5nZ\nWDNbEPysGnXO3Wa20MzmmdkZUeUdzWxmcGyIFWKGZQU0EZGQK8aprzKAnu5+LJF5eXubWTciM0KN\nc/fmwLhgHzNrRWSke2ugN/C8me2eMPQF4Doik2o0D44XSAFNRCTkius9NI/YPaNDcvBxItMeDgvK\nhwH9gu2+wFvunuHuS4CFQBczqwtUcvcJweQYw6POyZcCmoiIFFYNM5sS9bk+b4Vg4vgZwFpgrLtP\nBGq7+6qgymqgdrBdD4ieM25FUFYv2M5bXiANChERCTGD3OV/YiBtf3M5uns20M7MqgDvmVmbPMc9\nmP4w5hTQRERCriSmrXL3TWb2JZFnX2vMrK67rwq6E3evCLwSaBB1Wv2gbGWwnbe8QOpyFBGRmDCz\nmkFmhpmlAqcDc4HRwMCg2kBgVLA9GhhgZilm1pjI4I9JQffkFjPrFoxuvCLqnHwpQxMRCblifA+t\nLjAsGKmYAIxw9w/N7AdghJldAywD+gO4+ywzGwHMBrKAG4MuS4AbgNeAVGBM8CmQApqISMgVV5ej\nu/8EtN9H+Xrg1HzOGQwM3kf5FKDN3mfkT12OIiISCsrQRERCLMajHA9rytBERCQUlKGJiIRa6Vht\nOhYU0EREwkyz7YuIiJQuytBEREIuThI0BTQRkTCLjHKMj5CmLkcREQkFZWgiIiEXH/mZApqISPjF\nSURTl6OIiISCMjQRkZDTi9UiIhIKcTLIUV2OIiISDsrQRERCLk4SNAU0EZHQi5OIpi5HEREJBWVo\nIiIhZsTPKEdlaCIiEgrK0EREwiyO1kNTQBMRCbk4iWfqchQRkXBQhiYiEnZxkqIpoImIhJpplKOI\niEhpogxNRCTkNMpRRERKPSNuHqGpy1FERMJBGZqISNjFSYqmgCYiEnIa5SgiIlKKKEMTEQk5jXIU\nEZFQiJN4pi5HEREJB2VoIiJhFkcvoilDExGRUFBAExEJOYvR//Z7H7MGZvalmc02s1lmdmtQXs3M\nxprZguBn1ahz7jazhWY2z8zOiCrvaGYzg2NDzPY/tEUBTUQkxIzIKMdYfAohC/iju7cCugE3mlkr\nYBAwzt2bA+OCfYJjA4DWQG/geTNLDK71AnAd0Dz49N7fzRXQREQkJtx9lbtPC7a3AnOAekBfYFhQ\nbRjQL9juC7zl7hnuvgRYCHQxs7pAJXef4O4ODI86J18aFCIiEnIxHBNSw8ymRO0Pdfeh+7ynWSOg\nPTARqO3uq4JDq4HawXY9YELUaSuCssxgO295gRTQRETCLnYRLc3dO+33dmYVgHeAP7j7lujHX+7u\nZuYxa1EUdTmKiEjMmFkykWD2uru/GxSvCboRCX6uDcpXAg2iTq8flK0MtvOWF0gBTUQk5IpxlKMB\nrwBz3P2pqEOjgYHB9kBgVFT5ALP/b+/eg+0q6zOOfx9iUO6homCByjXKHQmBCOKkgFwsNxmh3EnJ\ngASlBVsQxU5xagrWqbY0AkVqiW0F4lQN1NKIdCqCSYGmhJuEiwGFIiTWIXJRMHn6x/seu91NyEk4\nZJ291vPJrDk7a6+91pvMnvNb72X9fnqjpG0piz/urMOTSyVNquc8reczK5Uhx4iIlluLuRz3B04F\n7pN0T933CeAyYJakqcATwPEAth+QNAt4kLJC8sO2l9XPnQNcC6wH3Fy3V5WAFhERI8L27ax8xu6g\nlXxmOjB9BfvvBnZdnesnoEVEtFxHMl8loEVEtF5HIloWhURERCukhxYR0WIl2X43umjpoUVERCuk\nh5/ecgEAAAuxSURBVBYR0WbDTyw88BLQIiJariPxLEOOERHRDumhRUS0XUe6aAloERGtNrw8jG3Q\n+YD23BPfXzL7zIlPNN2OAbAZsKTpRkQr5Lu0am9vugGDqPMBzfZbmm7DIJB093DqIEWsSr5La19W\nOUZExMATnZlCyyrHiIhoh/TQYriubroB0Rr5Lq1tHemiJaDFsNjOL6EYEfkurX1dWeWYIceIiGiF\n9NAiIlouqxwjIqIVOhLPMuQYr52knSQdKGls022JwSR1pQ8Rr6f00GIknABsDSyT9D3brzTdoBgs\ntg0gaRLwuO0fN9yk9uhQ+Zj00GIkfAp4HPhd4D3pqcVwSXqXpHXr6+2B6cAvm21VDKoEtFgjvUNE\ntpdTfhE9TYJarJ5LgJtqUFsEPAe8DCBpHUljGmxbi2iEttEtAS1WmyT1DBEdImkyMA74NPBDSlDb\nL0EtVkbSOgC2jwZ+CswCNqT09Nev7y0H1m2oia0hypDjSGyjXebQYrX1BLOPAh8AHgTOBK6x/WeS\nPgacBSwDbm+soTEq1Rui5fX1W2yfIGk2MJfynXmbpGXAWOBpSR+3/VKDTY4BkYAWa0TSwcBv2z5A\n0qXAPsCJkrD9GUnnA48228oYjXpuiH4f2FvSNNtHS7oKOAj4c2AMpde/MMHstRuAztWISECLYekd\nZqx+BJwraQowEXg/8HngEkljbX++gWbGgJD0AeB04AjbLwDYPlvSV4E/BY6xncUhI2QQhgtHQubQ\nYpX65sz2lbQpsMj248COwJW2nwbuBRYA9zTW2BgU2wE32n5a0tih+VbbxwHPAL/ZaOtiIKWHFqvU\nE8zOBi4AHgC+Jel64H5gpqS9gGMpd9zPNtbYGHVW0LsHeAo4QNLGtpfW444HnrQ9da03suW6kpw4\nAS1Wqq9n9lZgd8pc2d7A+4CpwAzKUut9gWNtP9ZQc2MU6vsOHQv8DHge+BZwMnCGpIWU+bKLgSOb\namurdSOeJaDFivX9IvoIsAWwi+2fAHPqsuuDgQuBv7L9L821NkarvgUgJ1FqoV0InENZCfsRyk3S\nm4ATbS9qqKnRAplDixXqu6s+HbgT2ErSDfX9m4HbKEurO3L/F2tC0ruAo4HJwFbAs8A1wL62L7Z9\nEnCa7fuaa2W7deOx6gS06NObAUTSBMqw0NW2bwR2AMZLug7A9mxgeu21RQAgaVxNY4Wk3YGXgBMp\nQe19tt8LfBG4QdIpALafb6q9bTdSD1UPwkrJDDnGr/QNM34Q2ImSxWGypDttL6iLP34g6VrbU4aW\nXEcASHoDMB44QtLbgM2Ak22/WFfHfqUe+j/A54B5zbQ02igBLX6lJ5gdRpnjOJQS1E4BjpK0vA4L\nbStp2+ZaGqNRvSH6ZV3k8Qng3cCFtl+sh7wBOFTSOyiLPybb/lFDze2UrqxyzJBj/Jqal3EacJft\nV2zfC8wGNgBOkrQLQCbvo1ftfR1W/zqekpPxC8Beko4EsD0D+BrlWcWjEszWoo5MoiWgddwKCisu\nomTN307SHgC27wD+FXiF8tBrRL+xwP6S5gKX2z4PuAJ4DDhS0gH1ZullYKbtB5prarxeJH1J0rOS\n7u/Z9xuSbpH0SP25ac97H5f0qKSFkg7t2T9B0n31vcuHWwA2Aa3D+ubMjpR0OCVDw7nAYuA4SbsB\n2P534FLbS5pqb4w+krYAqA/TPwPsDDxU9y0BbqJkjvkY8PeUnv/yZlrbXWuxg3Yt/9dTH3IRcKvt\nHYFb69+RtDOlOPAu9TNX9JQLupKS8HzHuvWfc4US0AJJ51CKdL4H+BJwft3GAVPqF48kiY1ekt4J\n/Lekv5R0EnAVZSXjYklX1BumRcAtwBnAJNsPN9jkeJ3Zvo2y4KfX0cDM+nomcEzP/utt/6J+Tx4F\n9qmLiTa2Pa/ecH+55zOvKgGtgyT9lqQNbLtmADmeshLtYmA/4GzgOErRzjGU54Yi+j0PfI8yRD2V\ncle9CTAHWArMkHQq5eZoqe2nmmpo1zW8bH/zmusV4MfA5vX1lpQk50OerPu2rK/7969SAlrHSNoc\n+ENgmqQN61DREmqVYNs/Bc4DdqtfwgsyzBgrYvtJygP3e1FWxN4KnErJln8T8GZgCjDD9s8bamag\nEfsDbCbp7p7trNVpSe1x9ef1HDEJaN2zGLiLMlf2e3Wy9VHg+voMEcDbKVlBxgAp4RH/T88k/UWU\nX1CbUXpqE4D7KPOwTwKn236wkUbG62GJ7b17tquH8Zln6jAi9efQiM9TwNY9x21V9z1VX/fvX6U8\nh9YRknYE1rG9UNI/UhIKHw6cafsiSVcCt0m6l5Jo+GTbyxpscoxidbh6KKg9AvwFJZidb/sbdX7t\nmdrjjwaJxrN83EhJn3dZ/Tm7Z/9XJH2OcoO9I3Cn7WWSlkqaBPwHcBrw18O5UAJaB0h6M7AQWCLp\nU5Qy91dT5jt2kPQh29Mk7UtJEvuZPGcWq1KHj16W9A/Ad4Av2P5Gfe+hRhsXjahp8SZThiafBP6E\nEshmSZoKPEGZs8f2A5JmAQ9SRoI+3HMTfQ5lxeR6wM11W6UEtA6w/RNJBwPfpgwz7wHcQJnUfxnY\nrd5t/53tXzTX0hhEtdd/EbCNpPV7MoNEx9g+cSVvHbSS46dTFp/1778b2HV1r5+A1hG2/60+uHg5\nJaBtDhxIeQ5kH+AdwHVAAlqsiXmUAq8xCg1CYuGRkIDWIbZvkfRHlCrTk2zPlHQjJcvD+rafa7aF\nMahsPyTphPTORqeu5HJMQOsY29+UtByYJ+ndKf0SIyXBLJqWgNZBtm+WtC7wbUkTkoooosUGpJbZ\nSEhA6yjbsyXdmmAW0W4Dkih/ROTB6g5LleCIaJP00CIi2q4jXbT00CIiohXSQ4uIaLmuLNtPDy0G\nlqRlku6RdL+kr0pa/zWca7Kkf66vj6qZL1Z27LhaQ251r3FJfQ5wWPv7jrlW0gdX41rb9FYNjm5r\nuHzMWpOAFoPsJdt72t6VksLr7N43Vaz2d9z2jbYve5VDxlFyzUXEKJKAFm3xXUqi5W0kLZT0ZUpG\nlK0lHSJprqT5tSe3IYCkwyQ9JGk+PWmbJE2RNKO+3lzS1yUtqNt+lGSr29fe4WfrcRdIukvSvTUB\n9NC5Lpb0sKTbKenFXpWkM+t5Fkj6p75e58G1BtXDko6ox4+R9Nmea3/otf5HRvtohLbRLgEtBl6t\n43Y4pQ4XlDIUV9jeBXgB+CRwsO29gLuBj0p6E/BF4EhK2ZMtVnL6y4Hv2N6DUsjyAUoNsMdq7/AC\nSYfUa+4D7AlMkPReSRMouTL3BN4PTBzGP+drtifW632fUgl6yDb1Gr8DXFX/DVOB52xPrOc/U9K2\nw7hOdElHIloWhcQgW0/SPfX1d4G/pdRVesL2vLp/ErAzcEct37UuMBd4J7DI9iMAtQTKiqrvHkip\nx0QtbfGcpE37jjmkbv9V/74hJcBtBHx9KCVUzZu5KrtK+jRlWHNDYE7Pe7Pqg/CPSPpB/TccAuze\nM7+2Sb32w8O4VkSrJKDFIHvJ9p69O2rQeqF3F3BLf1kLSb/2uddIwKW2/6bvGuetwbmuBY6xvUDS\nFEptqSH9petdr32u7d7Ah6Rt1uDa0VJZ5RjRDvOA/SXtACBpA0njgYco9bu2r8etrI7TrcC0+tkx\nkjYBfkbpfQ2ZA5zRMze3paS3ArcBx0haT9JGlOHNVdkIeFrSWODkvveOk7RObfN2lKKtc4Bp9Xgk\njZe0wTCuEx0xVLG6C6sc00OLVrO9uPZ0rpP0xrr7k7YflnQW8E1JL1KGLDdawSn+ALi6VttdBkyz\nPVfSHXVZ/M11Hm0nYG7tIT4PnGJ7vqQbgAXAs8Bdw2jyH1PKzi+uP3vb9EPgTmBj4GzbP5d0DWVu\nbX4t0roYOGZ4/zvRBfPn/+ec9cZqsxE63ZIROs/rQqWKekRExGDLkGNERLRCAlpERLRCAlpERLRC\nAlpERLRCAlpERLRCAlpERLRCAlpERLRCAlpERLRCAlpERLTC/wLh/JY8sXbohwAAAABJRU5ErkJg\ngg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f526761dcc0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = df.loc[:,'Actual'].values.astype(int),\n",
    "     pred_value = df.loc[:,'Prediction'].values.astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.678889Z",
     "start_time": "2017-07-24T00:04:21.672941Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Actual\n",
       "0.0    2152\n",
       "1.0    9698\n",
       "Name: Actual, dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_.groupby(by=\"Actual\").Actual.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.962382Z",
     "start_time": "2017-07-24T00:04:21.680232Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbQAAAGhCAYAAAAJL0FuAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xd4VNXWx/HvSkLv0qUIKgoI0gXpTUVpFlQURBT1+nqt\nqFiuYsV+LYjlol5FvYpYQYpiQUVA6UWa0kSK9N5J1vvHHGKIEAJMEubM78MzD3P2aWsgycraZ599\nzN0RERGJdQk5HYCIiEg0KKGJiEgoKKGJiEgoKKGJiEgoKKGJiEgoKKGJiEgoKKGJiEgoKKGJiEgo\nKKGJiEgoJOV0ACIiknUSC5/gvndHVI7lO9Z86e7tonKwLKCEJiISYr53B3lOvSQqx9o5/aUSUTlQ\nFlFCExEJNQOLj6tL8fEpRUQk9FShiYiEmQFmOR1FtlBCExEJO3U5ioiIxA5VaCIiYacuRxERiX0a\n5SgiIhJTVKGJiISduhxFRCTmGepyFBERiSWq0EREQs3U5SgiIiGhLkcREZHYoQpNRCTs1OUoIiKx\nTzdWi4iIxBRVaCIiYRZHj49RhSYiIqGgCk1EJOzi5BqaEpqISKhpUIiIiEhMUYUmIhJ2CfExKEQJ\nTUQkzDTbvoiISGxRhSYiEnZxch+aEpqISKhplKOIiEhMUYUmIhJ26nIUEZFQUJejiIhI7FCFJiIS\nZmZx0+WoCk1EREJBCU1EJOwsITqvzJzKbImZzTKz6WY2OWg7zsy+MrPfgr+Lpdn+HjNbYGbzzeyc\nNO31guMsMLP+ZocuM5XQJNTMLJ+ZfW5mm8zsw6M4TjczGx3N2HKKmTUzs/k5HYdko33djkf7yrxW\n7l7b3esHy3cD37h7FeCbYBkzqw50BU4D2gEvm1lisM8rwLVAleDV7lAnVUKTY4KZXW5mk81sq5mt\nNLNRZtY0CofuApQGirv7xUd6EHf/n7ufHYV4spSZuZmdnNE27j7W3U/NrphEgM7AoOD9IOD8NO2D\n3X2Xuy8GFgBnmFlZoLC7/+TuDrydZp+DUkKTHGdmvYHngceIJJ+KwEtApygc/gTgV3ffG4VjxTwz\n00CwuGPZ2uUIOPC1mU0xs+uCttLuvjJ4/yeR73OAcsAfafZdFrSVC96nb8+QEprkKDMrAjwM/NPd\nP3H3be6+x92Hu3ufYJs8Zva8ma0IXs+bWZ5gXUszW2Zmt5vZ6qC6uypY9xDQF7g0qPx6mdmDZvZu\nmvNXCqqapGC5p5ktMrMtZrbYzLqlaf8xzX6NzWxS0JU5ycwap1n3nZk9YmbjguOMNrMSB/n8++Lv\nkyb+883sPDP71czWm9m9abY/w8wmmNnGYNsBZpY7WPdDsNmM4PNemub4d5nZn8Cb+9qCfU4KzlE3\nWD7ezNaYWcuj+o+VY0v0uhxLBD0p+17XHeBsTd29NnAu8E8za552ZVBxeVZ8TCU0yWlnAnmBTzPY\n5l9AI6A2UAs4A7gvzfoyQBEiv8H1Al4ys2Lu/gCRqu8Ddy/o7m9kFIiZFQD6A+e6eyGgMTD9ANsd\nB4wIti0OPAuMMLPiaTa7HLgKKAXkBu7I4NRliPwblCOSgF8DugP1gGbA/WZWOdg2GbgNKEHk364N\ncAOAu+/7wVEr+LwfpDn+cUSq1f1+ALn7QuAu4F0zyw+8CQxy9+8yiFfi11p3r5/mNTD9Bu6+PPh7\nNZHv6zOAVUE3IsHfq4PNlwMV0uxePmhbHrxP354hJTTJacWJfJNk1CXYDXjY3Ve7+xrgIeCKNOv3\nBOv3uPtIYCtwpNeIUoAaZpbP3Ve6++wDbNMe+M3d33H3ve7+PjAP6Jhmmzfd/Vd33wEMIZKMD2YP\n0M/d9wCDiSSrF9x9S3D+OUQSOe4+JbiusNfdlwD/AVpk4jM9EFyn2JF+pbu/RuTaxc9AWSK/QEhY\n7HseWjZ0OZpZATMrtO89cDbwCzAMuDLY7EpgaPB+GNA16IWpTGTwx8Sge3KzmTUKRjf2SLPPQak/\nXXLaOiLdGEkZJLXjgd/TLP8etKUeI92+24GChxuIu28zs0uJVFNvmNk44HZ3n3eIePbFlLaP/8/D\niGeduycH7/clnFVp1u/Yt7+ZnUKkIqwP5CfyPTwlo88FrHH3nYfY5jUiP1yuc/ddh9hWYkq2zrZf\nGvg0GGGfBLzn7l+Y2SRgiJn1IvK9cgmAu882syFEfmnbS+TSw77vhRuAt4B8wKjglSElNMlpE4Bd\nREYwfXSQbVYQ6S7bVy1VDNqOxDYiiWCfMmlXuvuXwJdmlg94lMgP+mYHiSetisAXRxjT4XgFmAZc\n5u5bzOxWIiM5M5Lh9QozK0hkUM4bwINm9rG7r49KtBJX3H0RQW9CuvZ1RLrHD7RPP6DfAdonAzUO\n5/zqcpQc5e6biFw3eikYDJHfzHKZ2blm9lSw2fvAfWZWMhhc0Rd492DHPITpQHMzqxgMSLln3woz\nK21mnYOukl1Eui5TDnCMkcApFrnVICmo6qoDw48wpsNRCNgMbDWzqsD/pVu/CjjxMI/5AjDZ3a8h\ncm3w1aOOUo4t2X8fWo5QQpMc5+7/BnoTGeixhsgw3huBz4JNHgUmAzOBWcDUoO1IzvUV8EFwrCns\nn4QSgjhWAOuJXJtKnzD2/bbZAbidSJdpH6CDu689kpgO0x1EBpxsIVI9fpBu/YPAoGAU5CWHOpiZ\ndSZyw+q+z9kbqLtvdKeERPYO288xFhlBKSIiYZRQ9ATP0+LeQ2+YCTuHXT8lzewfxxxdQxMRCbsY\n6C6MBiU0EZEws2wd5Zij4uNTiohI6KlCExEJO3U5xoeixYp7mXIVDr2hyCEUyBP3304SJVOnTlnr\n7iVzOo5YE/ffgWXKVWDgJ9/mdBgSAmeceFxOhyAhkS+XpZ+J5qiYKjQREYl1RvwkNA0KERGRUFCF\nJiISZha84oASmohIqJm6HEVERGKJKjQRkZCLlwpNCU1EJOTiJaGpy1FEREJBFZqISMjFS4WmhCYi\nEmZxNGxfXY4iIhIKqtBERELM4ug+NCU0EZGQi5eEpi5HEREJBVVoIiIhpwpNREQkhqhCExEJuXip\n0JTQRETCTPehiYiIxBZVaCIiIacuRxERiXnxdGO1uhxFRCQUVKGJiIRcvFRoSmgiImEXH/lMXY4i\nIhIOqtBERMLM1OUoIiIhES8JTV2OIiISCqrQRERCLl4qNCU0EZEQ043VIiIiMUYVmohI2MVHgaYK\nTUREwkEVmohImOk+NBERCYt4SWjqchQRkVBQhSYiEnLxUqEpoYmIhF185DN1OYqISDioQhMRCTl1\nOYqISMwz09RXIiKHLTk5mUb163Bh5w6pbevXr6d9u7OoUa0K7dudxYYNGwBYt24d57RtRYmiBbn1\n5htzKmQJESU0EYmaAf1f4NRq1fZre+apJ2jZug2/zP2Nlq3b8MxTTwCQN29e+j74CI8/+UxOhBpX\n9lVpR/s61imhiUhULFu2jC9GjeCqq6/Zr33450PpfsWVAHS/4ko+H/YZAAUKFKBJ06bkzZs322ON\nN0poIiKH4c7bb6Xf40+RkLD/j5XVq1ZRtmxZAMqUKcPqVatyIjyJA0poInLURo4YTqmSpahbr16G\n28XKb/qhY1F6HeOU0ETkqE0YP47hw4dx6smV6NGtK9+N+ZarenQHoFTp0qxcuRKAlStXUrJUqZwM\nVUJMCU1Ejtoj/R5n4ZJlzF+whLf/N5iWrVrz5tvvAtC+QyfefWcQAO++M4gOHTvnZKhxKV6uoek+\nNBHJUnf0uZvul13CoDffoGLFE3j3/SGp6049uRJbNm9m9+7dfD7sM4aPHE216tVzMNoQ0uNjRESO\nTPMWLWneomXqcvHixRk1+psDbjt/wZLsCUrighKaiEiIGRAnBZoSmohIuMXG9a9o0KAQEREJBVVo\nceiJe25iwnejKVa8BG8NH5fa/uCtvfhj8QIAtm7ZRMFCRXhj6PesXLaUHuedScXKJwNQvVZ9bn/4\n3wC89tyjfPnZB2zdvIkvpi3N/g8jh+XUkytRqGAhEhMTAXj+xZc5s3Hjg25fomhB1m7celTnvPbq\nnowd+z1FChchISGB5/q/RKMzzzysYwz/fBhz587hzj53M2zoZ1Spckrq4JGHH+xL02bNad2m7VHF\nGWZxUqApocWjcy+8jAu7X8Njd92wX/uDz7+R+v6lJ+6nQMHCqcvlKlbijaHf/+1YjVudw4XdrqHb\nOWdkXcASVV98PYYSJUpk6zkfe+JpLryoC19/NZqbbvgHk6bNPKz9O3TsRIeOnQD4fOhnnNu+Q2pC\n6/vgw1GPN2zU5SihVatBYwoVKXbQ9e7OmFGf0bbDhYc81mm1G1C8VJlohifZbOvWrZx7dhvObFCX\n+rVr8vmwoX/bZuXKlbRt1ZyG9WpTr3YNfvxxLABffzWaFk3P5MwGdbm868Vs3ZpxNde0WXMWLoz0\nAsyYPp3mTRrRoM7pXNLlgtRZ+F96sT91Tq9Ogzqnc0W3rgC8M+gtbr35RiaMH8+I4cO49+47aViv\nNosWLuTaq3vyyccfMfrLL7i868Wp5/rh++9SZ/0/3DglNimhyd/MnDyB44qXpHylk1LbVi5bSq/O\nLbi5e0dmTJ6Qg9HJ0WrXthUN69WmWeOGQGTW+w8++pQJk6byxddjuLvP7bj7fvt8MPg9zjr7HH6e\nMp2JU2ZQq1Zt1q5dyxOPPcrIL79mwqSp1K1Xn/7PP5vhuUcM/5zTatQE4JqretDv8SeZNG0mNWrU\npN8jDwHwzNNP8NOkaUyaNpMXX3p1v/3PbNyY9h068dgTT/PzlOmceNJfX6Ot27Rl0sSf2bZtGwAf\nDfmAiy/pekRxhopFuhyj8TrWqctR/ubr4R/TpsNFqcvFS5VmyJgZFCl2HPN/mc6//nkFg0aM269L\nUmJH+i5Hd6fvffcybuwPJCQksGL5clatWkWZMn9V3vXrN+Af117Nnj176NjpfGrVrs3YH75n3tw5\ntG7eBIDde3bTsOGBr43de/edPPnYo5QoWZJXB77Bpk2b2LhpI82atwAis/B3C6qrmjVPp2ePbnTq\ndD4dO5+f6c+VlJTE2We3Y8Twz7nwoi6MGjWCfk88dVhxhpEBCQkxkI2iQAlN9rN3717GfjWCgZ/8\ndSNs7tx5yJ07DwCn1qhNuYqV+WPxQqrWrJNTYUoUDX7vf6xdu4bxE6eQK1cuTj25Ert27txvm6bN\nmvPVtz/wxcgRXNerJzff2puixYrRuu1ZvP3u+4c8x75raPts2rTpoNt+OmwEP479gRHDP+fJJ/ox\nedqsTH+Wiy/tyisvD+C4446jbr36FCpUCHfPdJwS29TlKPuZMv57Kp5YhVJlyqW2bVy/luTkZABW\n/LGEZUsWcnyFSjkUoUTbpk2bKFmyFLly5eL778aw9Pff/7bN77//TunSpbn6mmvpefU1TJs2lTMa\nNmLC+HEsXBC5JrZt2zZ++/XXTJ2zSJEiFCtaLPVa3Hv/e4emzVuQkpLCsj/+oEXLVvR7/Ek2bdr0\nt+tdBQsVYuuWLQc8brPmLZg+bSr/feM1Lr4kcv3taOIMi3jpclRCi0MP9b6WG7q2Y+niBXRpXoMR\nH76buu7bkZ/Qpv3+g0FmTBrP1Z2a0atzC/refBW9H/o3hYtGBpW88tSDdGleg507ttOleQ3efPHJ\nbP0scvS6Xt6NqVMmU792Tf73ztucWrXq37YZ+/13nFGvFo3q1+GjDz/gxptuoWTJkrz2xlv06H4Z\nDeqcTsumZzJ//rxMn/e1/w7i3rvupEGd05kxYzr33teX5ORkrrqyO/Vr16RRgzrccOPNFC1adL/9\nLr6kK889+zSN6tdh0cKF+61LTEzk3PM6MPqLUZzXPjIg5GjjDIPsnpzYzBLNbJqZDQ+WjzOzr8zs\nt+DvYmm2vcfMFpjZfDM7J017PTObFazrb5kIwNJf/I03VWvU9oGffJvTYUgInHHicTkdgoREvlw2\nxd3rR+VYZU/xk3q9FI1DMbvf2ZmKy8x6A/WBwu7ewcyeAta7+xNmdjdQzN3vMrPqwPvAGcDxwNfA\nKe6ebGYTgZuBn4GRQH93H5XReVWhiYiEWTaPcjSz8kB74PU0zZ2BQcH7QcD5adoHu/sud18MLADO\nMLOyRJLhTx6put5Os89BKaGJiEhmlTCzyWle1x1gm+eBPkBKmrbS7r4yeP8nUDp4Xw74I812y4K2\ncsH79O0ZUkI7xl1/8Vn06tyCi1ueTqdGp9Crcwt6dW7BymXRnWZq2e+LaHFqcT5777+pbf/uezuj\nhw7JYK/Dt3njBoa+/2bq8uqVy3nw1l5RPYdkXrPGDWlYrzZVTqxIhbIlaVivNg3r1eb3JUuy5HwP\n9r2PF194HoCrenRn2NDP/rbNVT26U7VK5dRY2rRsliWxxIvIbPtRu4a21t3rp3kN3O9cZh2A1e4+\n5WDxBBVXllzr0rD9Y9yrH34FwKhP3mP+L9O5te9TB9wuOTk5dX6+I3VciVJ8OOhVOlzSg6SkrPnS\n2LxpA8MGv0Xny64CoFTZcvtNuSXZa+z4n4HITBxTpkzm+f4DcjiiiKeeeY5OGdyDtnfv3v2+RtMv\nZ3a/+JCts+03ATqZ2XlAXqCwmb0LrDKzsu6+MuhOXB1svxyokGb/8kHb8uB9+vYMqUKLUXv37qV9\n/cq82O9erurYjLkzp9KleQ22bI7c3zN7+iR697wAgO3btvL43f/kH13a0uv8loz79osDHvO4EqU4\nvV4jRg/94G/rli1ZyB29unDtha25qVuH1EmMly1ZyPUXn0XPjk157blHaV+/MgDbtm7m1h6dueaC\nVlzVsRnjx3wJwMB/P8zSxQvo1bkF/3nmIZb9vohenSM31153YWuWLvot9Zw3XnYev82dlen4JXre\neG0gd/e5I3V54KuvcM9dd7JwwQLq1jqNK7p1pXbNanS77BJ27NgBwORJkzirdQsan1GPzh3OZdWq\nVVGN6cG+99GrZw9aNW/CtVf35M03Xufii87nnLat6HjeOaSkpNDnjt7Uq12D+rVr8snHHwHw7Tdf\nc3abllzYuQP169SMakyyP3e/x93Lu3sloCvwrbt3B4YBVwabXQnsm19tGNDVzPKYWWWgCjAx6J7c\nbGaNgtGNPdLsc1BKaDFs65bN1GpwJm9+PpYadRocdLtBLz3DGc3a8J+Pvub5QZ/x8pN92bVr5wG3\n7XbdLQx+YwApKSn7tT99f29ue+BpXvvkW67rfT/PP3IXAC88cjeXXn0jb33+I8VLlk7dPk+efPR7\n+R1e/3QMz771CQMevw+A627vS8XKJ/PG0O/5xx0P7HeOVuddwJhRka/Z1X8uZ/OmDVSpVvOw4pfo\nuPjSrgwb+il79+4F4O1Bb3Jlz6sBmDtnDjfedCvTZ80lb568vD7wP+zatYs7et/C+0M+ZvzEKXS9\nvDsPP3D/EZ+/zx23pXY59urZI7V9/vx5jBr9DW++HbnVZMb0aQz+8BNGjf6Gjz/6kPnz5jJxygyG\nf/EVfe64jdWrI4XA1CmTef7Fl5k+a+4RxxTLjoH70J4AzjKz34C2wTLuPhsYAswBvgD+6e7JwT43\nEBlYsgBYCGQ4whGysMvRzBx41t1vD5bvAAq6+4NZdc4DxPAWMNzdP8quc2anXLly0+ysDofcbvK4\nMfw89mveG/gCALt37WT1imVUCB4Hk1b5SidxcrWafDvy09S2LZs3MWfGZPre1DO1LTk58oNuzswp\nPPlapKJr26ELbzz/GBCZTuk/zzzMrCk/k5CQwJqVy9m4fl2GcbY693zuuf5yrvznHYwZ+Rkt23U+\n7PglOgoXLkzTps358otRVK58IomJiVStVo2FCxZQqXJlGjZqBMBl3brzxusDad6iJXPnzKb9OZFH\nuCQnJ1OufPmMTpGhg3U5duzUmbx586Yut217NsWKRW5pGj/uRy659DISExMpU6YMjZs0ZeqUyeTO\nnZuGjc6kYsWKRxxPrMuJ2fbd/Tvgu+D9OqDNQbbrB/Q7QPtkoMbhnDMrO5N3ARea2ePuvvZwdzaz\nJHffmwVxhUaevHn3+0JNTErCg8pq965dqe3uTr+X3qFcxcqZOu4V1/fmkduvo3rt+vsOQJFixx3w\n8TEH8+XQD9i2ZQuvfTqGpKQkujSvwe7dGVdVZcpVIF+BAixZMI9vR37KPU8MOKL4JTp6Xn0N/V94\nlhNOqESPK69KbU//w9HMcHdq1Dydb74bm6Ux5c9fYP/lAgUOsmW6/TK5ncS2rOxy3AsMBG5Lv8LM\nKpnZt2Y208y+MbOKQftbZvaqmf0MPGVmD5rZIDMba2a/m9mFZvZUcPf4F2aWK9ivr5lNMrNfzGxg\nZu4oD6My5Soyf/Z0AL4f/Xlqe4OmrfnknddSl3+dk/GzqCpXqcrxFSvx8w9fA1CoSFGKlyzND18N\nByAlJYUF834BoNrpdRkbtH8z4pPUY2zbsplixUuQlJTEpHFjWLMqMmI3f4GCbN928Ed3tD73At79\nz/Ps2b2bSidXPaL4JToaN2nC4oUL+eTjD+lyyaWp7UsWL2bypEkAfPD+ezRu3JRq1auzYsVyJk2c\nCMDu3buZM3t2tsbbpGkzPhwymJSUFFatWsWE8eOoWy8q9ybHtjiabT+rr6G9BHQzsyLp2l8EBrn7\n6cD/gP5p1pUHGrt772D5JKA10Al4Fxjj7jWBHURu3gMY4O4N3L0GkA84dD9cCPW8sQ/PPdSH6y5q\nQ65cudO038mOHdvp2bEpV7ZvzFuZmJ6qx//dzuqVfw0qeuC51xk2+C2u7tScK9s3ZkIwyOPmfz3O\ne6/156qOzfhz2VIKFIrMwH9250v4ZdpEenZsyrcjPk19FM1xJUpxymm16NmxKf955qG/nbdlu058\nM/xjWp3b+ajil+i44KIuNG3anCJF/voWrlqtGv1feJbaNauxfcd2el17HXny5OG9wR9x1529aVDn\ndBo1qMOkiT8f8XnTXkNrWK926lyiGbnwoi6ccmpVGtQ9nfbntOXJp5+lVKlSRxxDWER52P4xLcum\nvjKzre5e0MweBvYQSUAF3f1BM1sLlHX3PUGVtdLdSwTXvMa4+6DgGA8Ce9y9n5klBMfI6+4eHHe9\nuz9vZhcRuZEvP3Ac8GIwxcpbHOAaWnAz4HUApY8vX2/ImBlZ8m8QD3Zs30befPkxM0YPHcLYr0fw\nyIuDDr1jCIVx6qtO7dtx5133pD7mZeGCBVx+aRd+njI9hyMLt2hOfVWg3Kle9fpXD71hJkzt2zpq\ncWWF7Lgh43lgKvDmoTYMbEu3vAvA3VPMbI//lYFTgCQzywu8DNR39z+CJJiXDAQ3Aw6EyFyOmYxL\nDmDerGkMeOxfpKSkUKhIUe5+/MWcDkmiYN26dbRo2oi69eqnJjOJXTFQXEVFlic0d19vZkOAXsC+\naSjGE7lH4R2gG3A0V5L3Ja+1ZlYQ6AKEclTjsahOw6aHNVhEYkPx4sX5Ze5vf2s/6eSTVZ3FoFjo\nLoyG7LoP7d9AiTTLNwFXmdlM4ArgliM9sLtvBF4DfgG+BCYdRZwiIhKjsqxCc/eCad6vInJ9a9/y\n70QGeqTfp2e65QczOOaDad7fB9x3qOOJiMSjOCnQNJejiEiombocRUREYooqNBGREIvch5bTUWQP\nVWgiIhIKqtBEREItNmb5iAYlNBGRkIuTfKYuRxERCQdVaCIiIacuRxERiX0x8uiXaFCXo4iIhIIq\nNBGRENv3PLR4oIQmIhJy8ZLQ1OUoIiKhoApNRCTk4qRAU0ITEQk7dTmKiIjEEFVoIiJhpvvQRERE\nYosqNBGREDPNti8iImERJ/lMXY4iIhIOqtBEREIuIU5KNCU0EZGQi5N8pi5HEREJB1VoIiIhZhY/\nM4UooYmIhFxCfOQzdTmKiEg4qEITEQk5dTmKiEgoxEk+U5ejiIiEgyo0EZEQMyLzOcYDJTQRkZDT\nKEcREZEYogpNRCTMLH4eH6MKTUREQkEVmohIyMVJgaaEJiISZkb8PD5GXY4iIhIKqtBEREIuTgo0\nJTQRkbDTKEcREZEYogpNRCTEIg/4zOkosocSmohIyGmUo4iISAxRhSYiEnLxUZ9lkNDMrHBGO7r7\n5uiHIyIi0RYvoxwzqtBmA87+yX3fsgMVszAuERGRw3LQhObuFbIzEBERib7I1Fc5HUX2yNSgEDPr\namb3Bu/Lm1m9rA1LRESiInh8TDRex7pDJjQzGwC0Aq4ImrYDr2ZlUCIiIocrM6McG7t7XTObBuDu\n680sdxbHJSIiURIDxVVUZKbLcY+ZJRAZCIKZFQdSsjQqERGRw5SZCu0l4GOgpJk9BFwCPJSlUYmI\nSNTEwvWvaDhkQnP3t81sCtA2aLrY3X/J2rBERCQa4mmUY2ZnCkkE9hDpdtR0WSIicszJzCjHfwHv\nA8cD5YH3zOyerA5MRESiI16G7WemQusB1HH37QBm1g+YBjyelYGJiEh0HPupKDoy0324kv0TX1LQ\nJiIicszIaHLi54hcM1sPzDazL4Pls4FJ2ROeiIgcDbP4eR5aRl2O+0YyzgZGpGn/KevCERGRaIuT\nfJbh5MRvZGcgIiIiRyMzoxxPMrPBZjbTzH7d98qO4ERE5Ohl1yhHM8trZhPNbIaZzQ4m48DMjjOz\nr8zst+DvYmn2ucfMFpjZfDM7J017PTObFazrb5kIIDODQt4C3iQyUOZcYAjwQSb2ExGRY4BZdF6Z\nsAto7e61gNpAOzNrBNwNfOPuVYBvgmXMrDrQFTgNaAe8bGaJwbFeAa4FqgSvdoc6eWYSWn53/xLA\n3Re6+31EEpuIiEgqj9gaLOYKXg50BgYF7YOA84P3nYHB7r7L3RcDC4AzzKwsUNjdf3J3B95Os89B\nZeY+tF3B5MQLzex6YDlQKHMfT0REcpJh0RzlWMLMJqdZHujuA/c7X6TCmgKcDLzk7j+bWWl333e7\n159A6eB9OfYfaLgsaNsTvE/fnqHMJLTbgALAzUA/oAhwdSb2ExGRcFnr7vUz2sDdk4HaZlYU+NTM\naqRb72bmWRFcZiYn/jl4u4W/HvIpIiKxIPPXv6LK3Tea2Rgi175WmVlZd18ZdCeuDjZbDlRIs1v5\noG158D5X+ec2AAAgAElEQVR9e4YyurH6U4JnoB0k2AsPdXAREcl52TUPo5mVBPYEySwfcBbwJDAM\nuBJ4Ivh7aLDLMCLzAz9LZL7gKsBEd082s83BgJKfiUzB+OKhzp9RhTbgCD9TTPl10QrOubRvToch\nIXB86/NyOgSRnFYWGBRcR0sAhrj7cDObAAwxs17A70Seq4m7zzazIcAcYC/wz6DLEuAGIqPs8wGj\ngleGMrqx+psj/kgiInLMyK5nfrn7TKDOAdrXAW0Osk8/IuMz0rdPBmr8fY+Dy+zz0EREJAYZ8fPE\naj2sU0REQiHTFZqZ5XH3XVkZjIiIRF9CfBRomZrL8QwzmwX8FizXMrNDjjYREZFjQ4JF53Wsy0yX\nY3+gA7AOwN1nAK2yMigREZHDlZkuxwR3/z3dRcXkg20sIiLHjsjEwjFQXkVBZhLaH2Z2BuDBvQU3\nAXp8jIhIjIiF7sJoyEyX4/8BvYGKwCqgUdAmIiJyzMjMXI6riTyvRkREYlCc9DgeOqGZ2WscYE5H\nd78uSyISEZGoMYjm42OOaZm5hvZ1mvd5gQuAP7ImHBERkSOTmS7HD9Ium9k7wI9ZFpGIiERVvEwJ\ndSSfszJ/PW1URETkmJCZa2gb+OsaWgKwHrg7K4MSEZHoiZNLaBknNIvcjVeLv54UmuLuWfLobBER\niT4zi5tBIRl2OQbJa6S7JwcvJTMRETkmZeYa2nQz+9sD20REJDZEpr86+tex7qBdjmaW5O57iTx9\ndJKZLQS2Ebmtwd29bjbFKCIiRyFepr7K6BraRKAu0CmbYhERETliGSU0A3D3hdkUi4iIRJlmCoko\naWa9D7bS3Z/NgnhERCTK4iSfZZjQEoGCBJWaiIjIsSyjhLbS3R/OtkhERCT6TINCQJWZiEgoWJz8\nOM/oPrQ22RaFiIjIUTpohebu67MzEBERib7IKMecjiJ7ZOZ5aCIiEsPiJaHFy2NyREQk5FShiYiE\nnMXJjWiq0EREJBRUoYmIhJgGhYiISDjEyKNfokFdjiIiEgqq0EREQk6z7YuISMyLp2to6nIUEZFQ\nUIUmIhJycdLjqIQmIhJuRoJm2xcREYkdqtBERELMUJejiIiEgZ5YLWE2b8RDbNm2i+SUFPYmp9C0\n21P7rb/litY80ftCyre6i3Ubt5ErKZEB911G3eoVSfEU7njqY8ZO+Q2ALmfXpU+vc0hMTGDUD79w\nX/+hOfGRRESU0OJVu+teYN3GbX9rL1+6KG0aVWPpyr+e73r1hU0AaHDJY5QsVpDPBtxA0+5PU6xw\nfh679Xwad3uKtRu28trDV9DyjFP4buKv2fY5ROTQ4uXGag0Kkf08dcdF/OuFz3D31LaqJ5bhu0nz\nAVizYSubtuygXvWKVC5XnAVL17B2w1YAvv15Hue3qZ0jcYuIKKHFIXdnxKs3Me5/fVKrL4AOLWuy\nYvVGZv26fL/tZ/26nA4tapKYmMAJxxenTvUKlC9TjIV/rOGUSqWoWPY4EhMT6NSqFuVLF8vujyMi\nGdg3KCQar2OduhzjUJurnmPFmk2ULFaQ4a/eyPwlfzJ1zlL6XH0OHW4Y8LftBw2dQNXKpRn3vz4s\nXbmen2YsJjk5hY1bdnDzYx/w7pNXk+LOTzMWcWL5EjnwiUQkI/HS5aiEFodWrNkERLoPh307kwan\nVWLj5h2cUK44Ez+4B4BypYoy4b27aHbF06xat4U+//4kdf8xb/Xmt6WrARj5wy+M/OEXIHKtLTk5\nJZs/jYhIhBJanMmfNzcJCcbW7bvInzc3bc+symMDRzF7wQpOaHNP6nbzRjxEk25PsW7jNvLlzYVh\nbN+5m9YNq7I3OYV5i/4EoGSxgqzZsJWihfJx3SXN6N7nvzn10UTkIOKkQFNCizelihfig2evBSAp\nMZEPRk3mq/FzM9ynZLFCfP7yP0lJcVas2Uiv+walrnumTxdqnlIOgMcHfsGCoHITkWODET+DJZTQ\n4syS5etoeOkTh9yuavsHUt8vXbmeWhc8csDtrrznrWiFJiJyVJTQRETCzMDipM9RCU1EJOTiI53F\nT9eqiIiEnBLaMWreiIeYNORefhp8Nz8NvptGtSpnuP2acf8+6nMOfKg7C798lNy5IoV78aIFmDfi\noaM+bnodW55O1RPLpC7f/3/tadXw1KifR7JegsGw3k14rVf91LZzTy/DqDub8dvT51KzfJHU9lyJ\nxpOXns7IO5ox/PamNDzpuNR17WuXZcTtTRl1ZzP6tNfXQjQZkfvQovE61qnL8Rh2sPkWs1JycgpX\nnt+I1z78McvO0bHV6Ywa+0vq0P9HXhmRZeeSrNWzWWUWrtpGwbx//Sj59c8t3PDWVB7tUmO/bS9t\nVBGA854ZS/GCufnvNQ04/4VxFMmXi7s7VKXzc+NYv203T3c9ncZVijP+t3XZ+lnC7NhPRdGhhBZD\nCuTLzYfP/YOihfOTKymRh17+nOHfzdpvmzIlCvPOk1dTqEBekhITuOWxDxg3bSFtGlXl/v9rT+5c\nSSxetobrHniXbTt2/+0cA977jpu6tea/n4z/27rberThorPrkjtXEsPGzODRV0cCcPe17bjsvAas\n3bCVZas2MG3OHzz/zjdcdUFjel3UhFy5Eln0x1quvm8QtU4tT/sWNWlW72TuuqYdl93xOvdc245R\nY39h6/Zd9Dz/TLoF97I1q1eFW3u04aJbXs10/JJ9yhTJS6vqJXn564Vc3eKvHoSFqw/8S9jJpQsy\nYcFaANZt3c3mnXuoWb4IDixZu4312yL/n+N+W8s5NcsooclhU0I7hn0x8BaSU1LYvXsvzXs8w87d\ne7n09tfYsm0nxYsW4PtBd/wtoV16bn2+Gj+Xp974koQEI3/e3BQvWoC7r23Hef94ke07d3N7z7bc\nfEVrHh/4xd/O+cef6xk/bSGXtz+DkT/8dew2japyUsVSNO3+NGbGR8//gyZ1T2Lnzj2c36Y2Z1z6\nOLmSEpnw/l1Mm/MHAEO/ncGbn0YS4wM3dKDn+Y15ZfD3jPh+FqPG/sKnX0/f79zf/jyfl+67jPx5\nc7N95266nFOXD7+ccljxS/a5r3M1nhw+jwJ5MvdjZN6KzbQ5rTSfT1tJ2aJ5qVG+CGWL5mPCgrVU\nLlmAcsXy8eemnZxVowy5EuOlpsgeMdBbGBVKaMew9F2OZvDwjR1pUvdkUtw5vlQRShcvxKp1W1K3\nmTz7d/7zQHdyJSXy+ZgZzPx1Oc3qVaFq5TJ8+1ZvAHLnSuTnmYsPet6n3xzNh89dxxdjf0lta3tm\nNdqeWZWfBt8NQMF8eTi5YikK5c/D8O9msmv3Xnbt3ps6DRZA9ZPL8uANHShSKD8F8+fmqwnzMvy8\nyckpjB4/l/YtavLJ19M4t+lp/Ov5zw47fsl6raqVYt3W3fyybPN+18Iy8uHEZZxUqiCf3dqE5Rt2\nMHXJBlLc2bxjL30/nk3/K+qQ4s7UJRs4oUT+LP4E8cQ0bF+OPV3PbUCJYgVp3O1J9u5NYd6Ih8iT\nO9d+24ybupCzrnmedk1PY+DDV9D/3W/ZuHk73/48L9M3QS9cuoaZ85dz0dl1U9vM4On/juaNj8ft\nt+2Nl7c86HFee6g7l/R+jVm/Lqd7x4Y0r1/lkOf+8Msp/N+lzVm/aRtT5yxl6/ZdmHFY8UvWq1e5\nGG1OK0XLaiXJk5RIwbxJ/PvyWtz+3oyD7pOc4vQb9tesNB/edCaL10R+Yft2zmq+nROZZaZrowqk\npHl8kUhmaZRjDClSMB9rNmxl794UmtevwgnHF//bNhXLFmPVus28+el43vp0PHWqVmDirCWcWetE\nTqwQmQk/f97cnFyxVIbnevL1L7i1R5vU5a/Gz+XKzmdSIF9uAI4vWYSSxQoyYfoizmtekzy5kyiQ\nLzfnNvtrIEDB/Hn5c80mkpIS6Hpeg9T2rdt3UTB/3gOed+yU36hdrQJXX9iYD7+cAnBE8UvWembk\nfJo+MoYW/b7jlnenMWHBugyTGUDeXAnky50IQJNTSrA32VmwKvIsveIFI19XhfMl0a3xCXzw07Ks\n/QBxZN/UV9F4HetUocWQwaMm8fEL1zNpyL1MnbM0dZRgWs3qn8JtPdqwZ28y27bvotf977B2w1au\nfeBd3n78qtQh+Q+9PDzDeRfnLvqT6XP/oHa1CgB889M8qlYuw3eD7gBg245dXPWvQUyZs5QR389i\n0pB7Wb1uM7MXrGDT1h0APPzKcH545w7WbtjKpF9+p2D+PAB8+MUUXup7GTdc1oLL73xjv/OmpDij\nfviF7p0acU3fdwCOKH7JOWfXKE3fC6pzXMHcvH5Nfeas2MxVAydRvGAe3rquASkOqzbt5Pb3/7qG\nev/51alathAAA75awJK12Tu6V8LBPM5L+4T8pTzPqZfkdBgxrUC+3GzbsZt8eXPx1Ru3ceMj7zF9\nXvz9hn186/NyOgQJiUXPtp/i7vUPveWhnVS9lj/+3qhoHIpL65SLWlxZQRWaHLWX7r+cqieWIW/u\nJN4dPjEuk5nIsSw+hoQooUkU9Lz3rZwOQURECS1W/fD2HeTOncRxhfOTN28uVqyOPIX6ktsGsnTl\n+qif74EbOrBu41YGvPfd39p7dG7Emg1bU9vaXv0cW7fvinoMEj0f39yY3EkJFM2fizy5Eli1KfL/\ndf2bU1i+YUfUznNC8fyMvLMZi1ZvI1ei8dPC9Tz4yezDPs6b1zXgxkFTSUpI4LzaZXl/wlIAyhbN\nyz0dq3LzO9MPcYQ4ptn25VjXvMczAHTv2JB61Sty25Mf5lgszw36+m+JLq3ExASSk1MOupwRMyPe\nr/NmhYv6R254v6hBOWqUL8JDn8454HYJBilH+c+/aPU2Oj77I0kJxns3NKTNaaX4ZvbhDei5auAk\nAE4onofLz6yYmtBWbtypZHYIesCnxKyrL2xClRNKcc9znwJw7cVNqVyuBK9//CMfPvcPZi9YQc0q\n5Zi9YAXX9H2Hnbv2UK96RR7vfQEF8uVhzYYtXNf3XVav33KIM2Ws5wVncl6zGhQumI+UFOfpN0dz\n9zXt2Lp9FyeWL0Gdix6l95VtubzDGQC88fE4Xhn8PSdWKMFHz/+DGfOWUatqeTpcP4AVazYd9b+L\nZE5igjH54bZ8PGkZZ55cnPs++oUXe9Th3KfHsmXnXmpXLErvc0+hx38mkj93Ig9eeBonly5IUqLx\n/Be/pd5LdiB7U5xpv2/khOIFMIN7Olaj2SklcKD/6N/4YuaflC6ch/496pA/dxJJCca/PprF1CUb\n+fH+Vpz79FjubH8qJ5YqwOe9m/LD/DUM+ekPBlxZl47P/shntzbhtv9NT7237YMbG/HgJ7P5fe32\nw4pTYpcSWsh8+OUUfnr/Lu7rP5Tk5BR6pBn+Xv2ksvzfQ/9j4qwlvP7IFVxzURMGfvgjz/TpQpdb\n/8O6jdvoem59+t7QgRsffT/T57ztyrZ079QIgHUbt9L++gEA1KpagYaXPs7GLTto1fBU6lavSN2L\nHuWPPzfQoMYJXHpefZp2f5qkxATGvnMnP0z+jR27dnNqpdJcc/87TJ2zNPr/QHJIhfPlYuKi9Tw6\ndG6G29109sn8MG8NfQbPpHC+JD65pQk//rqW3XsPXH3ny51Io5OL89SI+ZxXqywnlypA+3+P5biC\nufns1iZMWrSezvXK8c3s1Qwcs4gEg7y5Evc7xtMj5nNCiQJ0fDYyefYJxf+aUWTE9BW0r1WWAV8v\noEyRvBTJl4u5K7ZwV4dTDyvOMFKXYxYxs/OBT4Fq7j7PzCoBjd39vWB9beB4dx95hMdfAtR397XR\niTi2bNm2kx+nLuScJtVZvGwtySnO/MWrOLFCCRYvW8vEWUsAeH/EJHpd1IQfJv9GtRPLMuLVmwBI\nTEhg+eoNh3XOg3U5fj1hLhu3/HU95ueZi/njz8ixG9c5ic++mc7OXXsA+Py7mTSpcxJf/zSXRcvW\nKpnloF17kxk9a9Uht2t6SklaVC3FP1qfBECepASOL5rvb/eQ7auoUtz5atYqxv26lgcuqM7n01aS\n4rB2y24mL95AzfJFmPXHJh7tUoM8uRL4atYq5q3MfE/BiBkree3q+gz4egHta5dl1Mw/DyvOMIuP\ndJYzFdplwI/B3w8AlYDLgfeC9bWB+sARJTSBtz4dz81XtOb3Fet4e+hPqe3pr0W5O2bGL78tp22v\n56Mex/Z0s+Fv35G5gSKaRT9n7dqzf+WSnOKpz8LKk+uvqzFmkUEkS9dtz/B4+66hZcaEBeu4/OWf\naFW9FM9cXouBYxYxbOqKTO27YsNOtu9O5uTSBWlfuyx9Bs88rDjl6JlZBeBtoDTgwEB3f8HMjgM+\nIPLzfglwibtvCPa5B+gFJAM3u/uXQXs94C0gH5F8cIsf4oJ6tl4rNLOCQFMiwXcNmp8AmpnZdDO7\nC3gYuDRYvtTMzjCzCWY2zczGm9mpwbESzewZM/vFzGaa2U3pzpXPzEaZ2bXZ+BGPCRNmLKJy+RJc\n2LYOH42ektpeqVxx6lWPPJPq0nPrM37aIuYu+pPjSxWl/mknAJArKZFqaR6+mVXGTV1Ip1a1yJsn\nFwXy5aZDy5qMm7Ywy88rh2/Z+h3UqBB5UGe70//62hg7fw09mp6Quly9XOFMH3PSovV0qFMWs8i0\nV/UqFWPWsk0cXywva7bsYvBPf/DRxGWclu6YW3ftpUCexIMcFUZMX8n1rU8id1JC6rRaRxNnWJhF\n55UJe4Hb3b060Aj4p5lVB+4GvnH3KsA3wTLBuq7AaUA74GUz2/cf/ApwLVAleLU71Mmzu0LrDHzh\n7r+a2bogA98N3OHuHQDMbBWRLsMbg+XCQDN332tmbYHHgIuA64hk+9rBurRTfhcEBgNvu/vb2fXh\njiWffj2NUyuVZvPWnalt8xav4uYrWnP6KeWZvWAFb3wyjt179nL5na/z7z4XU6hAXhITE3jhnW+Y\ne4BptQ4m7TU0gC63vHrIfSbP/p0Pv5jCj+/eCcBrH/7I7AUrUudrlGNH/9G/8djFNdm8Yw8TF61P\n076A+ztXY+QdzTCD39du5/o3p2RwpL+MmvkntU8oxsjbm+FAv2FzWbd1N13OKE+v5pXZk5LCtp3J\n+02PBaTO8D/yjmaMmbuaIT/9sd/6kTNWcm+najz/xa9RiTMMIqMcs6fT0d1XAiuD91vMbC5QjsjP\n/pbBZoOA74C7gvbB7r4LWGxmC4AzgktHhd39JwAzexs4H8hwypNsnfrKzIYDL7j7V2Z2M1ARGM7+\nCa0n+ye0CkB/IhnagVzuXtXMPgZedfev0p1jCbAJeMrd/3eQOK4jkhAhV8F6eU+7MtofNccNHXAD\nT785mh+nLADgxAoleO/pa2jU9Ykcjiy8NPWVREs0p76qclotf3bw6Ggcik6nl/kdSDs+YaC7DzzQ\ntsH4iB+AGsBSdy8atBuwwd2LmtkA4Cd3fzdY9waRpLUEeMLd2wbtzYC79uWJg8m2Ci2ooFoDNc3M\ngUQiCWrEIXZ9BBjj7hcE/0DfZeJ044B2Zvbegfpcg/+AgRCZyzGznyEWHFekAN+/fTtT5yxNTWYi\nEt+iOMhxbWYSbXB56WPgVnffnHaUpbt7kAOiLjuvoXUB3nH3E9y9krtXABYDKUChNNttSbdcBFge\nvO+Zpv0r4B9mlgSpCXOfvsAG4KWofoIYsH7TNmp2fvhvzw5b9MdaVWciccmi9idTZzPLRSSZ/c/d\nPwmaV5lZ2WB9WWDfjYDLgQppdi8ftC0P3qdvz1B2JrTLiAzXT+tjIhcEk81shpndBowBqu8bFAI8\nBTxuZtPYv6J8HVgKzDSzGURGSqZ1C5DPzJ7Kgs8iIiLpBN2JbwBz3f3ZNKuGAfuu7VwJDE3T3tXM\n8phZZSKXliYG1+I2m1mj4Jg90uxzUNnW5ejurQ7Q1v8gmzdIt3xKmvf3BfvuBXoHr7THrJRm8arD\nDlREJGSy8b7qJsAVwCwz2zei514io9mHmFkv4HfgEgB3n21mQ4A5REZI/tPdk4P9buCvYfujOMSA\nENBMISIiEiXu/iMHv4+7zUH26Qf0O0D7ZCIDSjJNCU1EJMSyc9h+TlNCExEJs8zfFB3z4uWpAiIi\nEnKq0EREQi5eKjQlNBGRkMvsPWSxTl2OIiISCqrQRERCzICE+CjQlNBERMJOXY4iIiIxRBWaiEjI\naZSjiIiEgrocRUREYogqNBGREIunUY6q0EREJBRUoYmIhFrmnzYd65TQRETCTLPti4iIxBZVaCIi\nIRcnBZoSmohImEVGOcZHSlOXo4iIhIIqNBGRkIuP+kwJTUQk/OIko6nLUUREQkEVmohIyOnGahER\nCYU4GeSoLkcREQkHVWgiIiEXJwWaEpqISOjFSUZTl6OIiISCKjQRkRAz4meUoyo0EREJBVVoIiJh\nFkfPQ1NCExEJuTjJZ+pyFBGRcFCFJiISdnFSoimhiYiEmmmUo4iISCxRhSYiEnIa5SgiIjHPiJtL\naOpyFBGRcFCFJiISdnFSoimhiYiEnEY5ioiIxBBVaCIiIadRjiIiEgpxks/U5SgiIuGgCk1EJMzi\n6EY0VWgiIhIKqtBEREIuXobtK6GJiISYET+jHNXlKCIioaAKTUQk5OKkQFNCExEJvTjJaOpyFBGR\nUFCFJiISchrlKCIioaBRjiIiIjFEFZqISMjFSYGmhCYiEnpxktHU5SgiIqGgCk1EJMQik+3HR4mm\nCk1EREJBFZqISJhZ/AzbV0ITEQm5OMln6nIUEZFwUIUmIhJ2cVKiKaGJiISaxc0ox7hPaL5jzdqd\n01/6PafjiAElgLU5HcSxbNH0l3I6hFihr6VDOyGnA4hFSmjuJXM6hlhgZpPdvX5OxyGxT19L2S+7\nRjma2X+BDsBqd68RtB0HfABUApYAl7j7hmDdPUAvIBm42d2/DNrrAW8B+YCRwC3u7oc6vwaFiIiE\nmEXxlQlvAe3Std0NfOPuVYBvgmXMrDrQFTgt2OdlM0sM9nkFuBaoErzSH/OAlNBERCQq3P0HYH26\n5s7AoOD9IOD8NO2D3X2Xuy8GFgBnmFlZoLC7/xRUZW+n2SdDcd/lKJk2MKcDkNDQ11J2y9kxIaXd\nfWXw/k+gdPC+HPBTmu2WBW17gvfp2w9JCU0yxd31Q0iiQl9L2S+KoxxLmNnkNMsDD+f/093dzA55\nLexIKaGJiEhmrT2CAT2rzKysu68MuhNXB+3LgQpptisftC0P3qdvPyRdQxMRCTmz6LyO0DDgyuD9\nlcDQNO1dzSyPmVUmMvhjYtA9udnMGpmZAT3S7JMhVWgiIiGXXZfQzOx9oCWRrsllwAPAE8AQM+sF\n/A5cAuDus81sCDAH2Av8092Tg0PdwF/D9kcFr0NSQpOjZmbVgLLAWHffk9PxSOwxM8vMfUZybHP3\nyw6yqs1Btu8H9DtA+2SgxuGeXwlNoqErkb7wZDMbr6Qmh2tfMjOzRsASd/8zh0MKjzh6fIyuoUk0\nPERkBoBLgaZmlitnw5FYYWZ1zCx38P4kIr+t783ZqCRWKaHJEQku1gLg7ilEfhCtRElNDs+DwOdB\nUlsMbAJ2A5hZQpqZI+SoZONcITlICU0OW9rrHWZ2tpm1BIoCjwJLiSS1xkpqcjBmlgDg7p2BDcAQ\noCCRSj9/sC4FyJ1DIYaGkeOjHLONrqHJYUuTzHoDFxAZpXQt8Lq7P2ZmdwHXEZlw9MccC1SOScEv\nRCnB+5Lu3tXMhgITiHzNlDWzZCAXsNLM7nH3HTkYssQIJTQ5ImbWFmjl7s3M7HHgDOAyM8PdnzSz\n24jMzSaynzS/EN0M1Dez/3P3zmb2KpHRcE8BiUSq/vlKZkcvBoqrqFBCk0w5wLDqP4CbzKwn0AA4\nD3gOeNDMcrn7czkQpsQIM7uAyE22Hdx9G4C7X29mHwKPAOe7uwaHREksdBdGg66hySGlu2bW0MyK\nAYvdfQmRu/tfCe7unwnMAKbnWLASK04EhgXTIeXad73V3S8GVgHH52h0EpNUockhpUlm1wN3ArOB\n0WY2GPgFGGRmdYELifzGvfqgB5O4c5CbppcDzcyssLtvDra7BFjm7r2yPciQi+LkxMc0JTQ5qHSV\nWSngdCLXyuoDZxF50uwAIkOtGwIXuvvCHApXjkHpvoYuBLYAW4HRQDfgajObT+R62b+AjjkVa6jF\nRz5TQpMDS/eD6EagDHCau68DvgyGXbcF+gAvuPvInItWjlXpBoBcTuRZaH2IzNV3HXAjkV+S8gKX\nBQ96FDkiuoYmB5Tut+orgYlAeTP7IFg/CviByNDqOPn9T46EmdUh8nTilkQeBbIaeB1o6O7/cvfL\ngR7uPivnogy3+LitWglN0kk7A4iZ1SPSLTTQ3YcBJwOnBDNq4+5DgX5B1SYCgJkVDaaxwsxOB3YA\nlxFJame5e3PgNeADM+sO4O5bcyresIvWTdWxMFJSXY6SKl03YxegGpFZHFqa2UR3nxEM/lhkZm+5\ne899Q65FAMwsCTgF6BA8zLEE0M3dtwejY98LNl0PPAv8lDORShgpoUmqNMmsHZFrHOcQSWrdgU5m\nlhJ0C1UOHsgnkir4hWhvMMjjXuBMoI+7bw82SQLOMbNTiQz+aOnuf+RQuHElXkY5qstR9hPMy/h/\nwCR33+PuM4k8LbYAcLmZnQagi/eSVlB9tQsWTyEyJ+NLQF0z6wjg7gOAT4jcq9hJySwbxclFNFVo\nce4A9wgtJjJr/olmVsvdZ7j7uODG19ZEbnoVSS8X0MTM+gK4+5lmVoLIyMaOZraRyHRWu4H3983l\nKBJNSmhxLN01s45EnkO1EbgJeAG4eF83o7t/Z2Y/a149ScvMyrj7n+6+2sxWAdWJVGG4+1oz+5zI\n19VdQC2gjZJZ9ouB4ioq1OUomNkNRB7S2RT4L3Bb8CoK9DT7//buNcSu6gzj+P8xaJubUSymIOrU\ne62XYEhNLYiUEC+tkg8qWi9NDdHMh1JtGwhoQaGgRQpF4qVW8YJiVbwrMsQUvDWp0dFY2+bSVlqs\n0szy090AAAWpSURBVE5KifUO9vHDWgPHwZiZ5OiZs/fzGw45c/Y6e60zBN6z9l7rfXU4QIJZdJJ0\nGPC6pF9K+i5wA2Ul44ik6+oXpleBVcAFwHzbm3o45Gi4BLQWkrSfpOm2XTOAnElZiXYpcBywDDiD\nUrRzCmXfUMRYbwG/o1yiXgJcD8wChoA3gZWSzqN8OXrT9j97NdC2a8uy/QS0lpE0G/gxMChpRs27\nuIVaJdj2f4GLgSNrwuHltrf0bMAxadl+jbLh/hjKitjVwHmUbPmPAHsBi4GVtt/r0TADde1nsktA\na58RYB0lm/n360bqvwC/qXuIAPanZAWZQrn/EfExHRvwVwCm7Dd7A5gL/IFyH/Y14Hu2/9STQUbr\nZFFIS0g6GNjF9kZJd1ISCp8MLLW9QtL1wFOSXqYkGj7H9oc9HHJMYvVy9WhQ2wz8ghLMLrH9YL2/\n9q86448eEv1xubAbEtBaQNJewEZgi6QrKGXub6Tc7zhI0kW2ByUdS0kS+/PsM4vtqStkP5B0B/Ak\ncK3tB+uxDT0dXLRSAloL2P6PpAXAE5TLzEcDd1Nu6n8AHFm/bd9i+/3ejTT6UZ31rwAGJE3ryAwS\n8blKQGsJ27+VdCJwDSWgzaZslD6LUr7jUOAuIAEtdsRaSoHXmIRyyTEax/YqST+hVJmeb/s2SQ9T\nsjxMs721tyOMfmV7g6SzMjubnPphhWI3JKC1jO3HJP0fWCvpGyn9Et2SYBa9loDWQrYfl7Qb8ISk\nuUlFFNFgfbIpuhsS0FrK9kOSVieYRTRbnyTK74psrG6xVAmOiCbJDC0ioulaMkXLDC0iIhohM7SI\niIZry7L9zNCib0n6UNJLkl6RdK+kaTtxrhMkPVqfn1YzX2yr7R61htxE+7i87gMc1+tj2twq6fQJ\n9DUg6ZWJjjGaKeVjIia/d23PsX0EJYXXss6DKib8f9z2w7av+pQmewATDmgR8dlKQIumeJqSaHlA\n0kZJt1MyouwraaGkNZKG60xuBoCkkyRtkDRMR9omSYslrazPZ0t6QNL6+jgOuAo4sM4Or67tlkta\nJ+nlmgB69FyXStok6RlKerFPJWlpPc96SfeNmXUukPR8Pd93avspkq7u6Puinf1DRvOoS4/JLgEt\n+l6t43YypQ4XwMHAdba/BrwNXAYssH0M8DzwI0lfBH4NnEope/LlbZz+GuBJ20dTCln+kVID7K91\ndrhc0sLa59eBOcBcScdLmkvJlTkHOAWYN46Pc7/tebW/P1MqQY8aqH18G7ihfoYlwFbb8+r5l0r6\nyjj6iTZpSUTLopDoZ1MlvVSfPw3cTClc+nfba+vr84HDgWdr+a7dgDXAYcCrtjcD1BIoF35CH98C\nzgeo9eG2StpzTJuF9fFi/X0GJcDNBB4YTQlV82ZuzxGSfka5rDkDGOo4dk/dCL9Z0t/qZ1gIHNVx\nf21W7XvTOPqKaJQEtOhn79qe0/lCDVpvd74ErLJ99ph2H3vfThJwpe1fjenj4h04163AItvrJS0G\nTug45jFtXfv+ge3OwIekgR3oOxoqqxwjmmEt8E1JBwFImi7pEGADpX7XgbXd2dt4/2pgsL53iqRZ\nwP8os69RQ8AFHffm9pG0N/AUsEjSVEkzKZc3t2cm8IakXYFzxhw7Q9IudcwHUIq2DgGDtT2SDpE0\nfRz9REuMVqxuwyrHzNCi0WyP1JnOXZK+UF++zPYmSRcCj0l6h3LJcuYnnOKHwI2SllAqfQ/aXiPp\n2bos/vF6H+2rwJo6Q3wLONf2sKS7gfXAv4F14xjyT4HfAyP1384x/QN4DtgdWGb7PUk3Ue6tDdci\nrSPAovH9daINhodfGJq6q77UpdNt6dJ5PhMqVdQjIiL6Wy45RkREIySgRUREIySgRUREIySgRURE\nIySgRUREIySgRUREIySgRUREIySgRUREIySgRUREI3wEl3sp0Gz/DOQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f5258d90278>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot(actual_value = df_.loc[:,'Actual'].values.astype(int),\n",
    "     pred_value = df_.loc[:,'Prediction'].values.astype(int))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2017-07-24T00:04:21.986741Z",
     "start_time": "2017-07-24T00:04:21.963899Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "no_of_features  hidden_layers\n",
       "1               1                (-0.0106932141878, 0.512292833118)\n",
       "                3                                        (nan, nan)\n",
       "12              1                  (0.553520322795, 0.771103258353)\n",
       "                3                  (0.589579017435, 0.761202912378)\n",
       "24              1                   (0.553882891437, 1.00945197776)\n",
       "                3                 (0.0413605111368, 0.914544769148)\n",
       "48              1                   (0.86589960535, 0.870006020745)\n",
       "                3                  (0.221270489485, 0.881508919623)\n",
       "122             1                   (0.372951841349, 1.01979176516)\n",
       "                3                  (0.750740841232, 0.871289928235)\n",
       "dtype: object"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from scipy import stats\n",
    "\n",
    "def fn(x):\n",
    "    #print(x)\n",
    "    return stats.norm.interval(0.95, loc=x.f1_score.mean(), scale=x.f1_score.std())\n",
    "psg.apply(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "_draft": {
   "nbviewer_url": "https://gist.github.com/7d1ace18a82178e15ece8fc5252fce88"
  },
  "anaconda-cloud": {},
  "gist": {
   "data": {
    "description": "Hyper parameter tuning",
    "public": false
   },
   "id": "7d1ace18a82178e15ece8fc5252fce88"
  },
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
